diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index b1ec8340f..746dca51f 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -474,6 +474,8 @@ Implement and verify in dependency order: - [x] Implement the immutable effective-QP value used by reconstruction, including independent luma/chroma bit-depth offsets, the normative 4:2:0 mapping plateaus, the 4:2:2/4:4:4 saturation rule, and combined picture/slice/coding-unit chroma offsets. - [ ] Select each transform unit's coding-unit luma QP and chroma-adjustment-list entry, then pass the derived component QP into inverse quantization. - [ ] Implement transform skip, coefficient rotation, implicit and explicit residual DPCM, transquant bypass, and lossless reconstruction. + - [x] Implement allocation-free SIMD-first transform-skip normalization, complete-block coefficient rotation, transquant-bypass copying, implicit intra-direction selection, and horizontal/vertical inverse residual DPCM with a scalar fallback and signed residual clipping. + - [ ] Decode explicit inter residual-DPCM modes and connect bypass, transform skip, residual DPCM, prediction addition, and lossless reconstruction through transform-unit traversal. - [ ] Connect coefficient decoding, inverse quantization, transform selection, reusable scratch, and add/clip to transform-unit traversal. - [ ] Deblocking and sample-adaptive offset for every signaled luma/chroma and bit-depth path. - [ ] Tiles, wavefront entry points, dependent slices, and all other parallelization syntax permitted by the exposed still-image profiles. @@ -572,6 +574,7 @@ Tasks: - [x] Add a permanent frame-wide HEVC reference-preparation benchmark. On .NET 10, complete and partially substituted borders measured 465.3 and 566.7 microseconds per 2,040-block padded 1920x1088 frame, compared with forced-scalar timings of 477.5 and 608.0 microseconds, with zero managed allocations. - [x] Add a permanent frame-wide HEVC inverse-transform benchmark. On .NET 10, dense-coefficient 32x32 twelve-bit inverse DCT, transposition, and add/clip measured 3.643 milliseconds per padded 1920x1088 frame, compared with 45.23 milliseconds with hardware intrinsics disabled: 12.4 times faster with zero managed allocations. - [x] Add a permanent frame-wide HEVC inverse-quantization benchmark. On .NET 10, dense 32x32 twelve-bit flat and scaling-list paths measured 69.98 and 238.4 microseconds per padded 1920x1088 frame, compared with forced-scalar timings of 1.469 and 1.609 milliseconds: 21.0 and 6.7 times faster with zero managed allocations. Pre-expanding the scaling matrices once reduced the SIMD scaling-list path from 4.974 milliseconds to 238.4 microseconds. + - [x] Add a permanent frame-wide HEVC residual-reconstruction benchmark. On .NET 10, dense 32x32 twelve-bit transform skip, horizontal RDPCM, and vertical RDPCM measured 79.64, 275.61, and 110.81 microseconds per padded 1920x1088 frame, compared with forced-scalar timings of 571.5 microseconds, 1.512 milliseconds, and 1.257 milliseconds: 7.2, 5.5, and 11.3 times faster with zero managed allocations. - [ ] Implement vector paths only for confirmed hot loops, using existing `Vector128`, `Vector256`, and `Vector512` helper and dispatch patterns where supported. - [ ] Prioritize shared color conversion and pixel packing, chroma upsampling, inverse-transform add-and-clip, intra predictors, HEVC deblock/SAO, AV1 loop filter/CDEF/restoration, and contiguous grid copies. - [ ] Benchmark the complete decode color pipeline on representative 8/10/12-bit AVIF and HEIC images with and without embedded ICC profiles. Report absolute end-to-end timings and allocations in addition to the isolated YUV/CICP and ICC stage costs. diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualDpcmMode.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualDpcmMode.cs new file mode 100644 index 000000000..49cc61360 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualDpcmMode.cs @@ -0,0 +1,25 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Identifies the differential pulse-code modulation applied to an HEVC residual block. +/// +internal enum HevcResidualDpcmMode : byte +{ + /// + /// No residual differential pulse-code modulation is applied. + /// + None = 0, + + /// + /// Residual differences accumulate from left to right within each row. + /// + Horizontal = 1, + + /// + /// Residual differences accumulate from top to bottom within each column. + /// + Vertical = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs new file mode 100644 index 000000000..caa99a5b8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs @@ -0,0 +1,555 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Reconstructs HEVC transform-skipped, bypassed, and differential residual blocks. +/// +internal static class HevcResidualReconstructor +{ + /// + /// The horizontal intra-prediction mode defined by H.265. + /// + private const int HorizontalIntraPredictionMode = 10; + + /// + /// The vertical intra-prediction mode defined by H.265. + /// + private const int VerticalIntraPredictionMode = 26; + + /// + /// The minimum residual sample represented by the decoder reconstruction pipeline. + /// + private const int ResidualMinimum = short.MinValue; + + /// + /// The maximum residual sample represented by the decoder reconstruction pipeline. + /// + private const int ResidualMaximum = short.MaxValue; + + /// + /// Defines a closed transform-skip normalization operator for every SIMD width and the scalar tail. + /// + private interface ITransformSkipOperator + { + /// + /// Normalizes sixteen transform-skipped coefficients. + /// + /// The dequantized coefficients. + /// The nonnegative shift magnitude. + /// The reconstructed residuals. + static abstract Vector512 Invoke(Vector512 values, int shift); + + /// + /// Normalizes eight transform-skipped coefficients. + /// + /// The dequantized coefficients. + /// The nonnegative shift magnitude. + /// The reconstructed residuals. + static abstract Vector256 Invoke(Vector256 values, int shift); + + /// + /// Normalizes four transform-skipped coefficients. + /// + /// The dequantized coefficients. + /// The nonnegative shift magnitude. + /// The reconstructed residuals. + static abstract Vector128 Invoke(Vector128 values, int shift); + + /// + /// Normalizes one transform-skipped coefficient. + /// + /// The dequantized coefficient. + /// The nonnegative shift magnitude. + /// The reconstructed residual. + static abstract int Invoke(int value, int shift); + } + + /// + /// Gets the 4:2:2 chroma intra-angle remapping defined by H.265 Table 8-4. + /// + private static ReadOnlySpan Chroma422IntraAngleMap => + [ + 0, 1, 2, 2, 2, 2, 3, 5, 7, 8, 10, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 23, 24, 24, 25, 25, 26, 27, 27, 28, 28, 29, 29, 30, 31, + ]; + + /// + /// Copies one transquant-bypass coefficient block into residual sample order. + /// + /// The decoded coefficients in raster order. + /// The destination residual block in packed raster order. + /// Whether the complete coefficient order is reversed. + public static void CopyBypassed(ReadOnlySpan coefficients, Span residual, bool rotate) + { + Span destination = residual[..coefficients.Length]; + if (!rotate) + { + coefficients.CopyTo(destination); + return; + } + + CopyReversed(coefficients, destination); + } + + /// + /// Reconstructs one transform-skipped residual block from dequantized coefficients. + /// + /// The dequantized coefficients in raster order. + /// The destination residual block in packed raster order. + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component precision. + /// The transform dynamic range excluding its sign bit. + /// The base-two logarithm of the equivalent square transform size. + /// Whether transform-skip precision is extended by the sequence. + /// Whether the complete coefficient order is reversed. + public static void ApplyTransformSkip( + ReadOnlySpan coefficients, + Span residual, + int width, + int height, + int bitDepth, + int maxTransformDynamicRange, + int equivalentLog2TransformSize, + bool extendedPrecisionProcessingEnabled, + bool rotate) + { + int transformShift = maxTransformDynamicRange - bitDepth - equivalentLog2TransformSize; + if (extendedPrecisionProcessingEnabled) + { + transformShift = Math.Max(0, transformShift); + } + + int coefficientCount = width * height; + if (transformShift >= 0) + { + ApplyTransformSkip(coefficients[..coefficientCount], residual[..coefficientCount], transformShift, rotate); + } + else + { + ApplyTransformSkip(coefficients[..coefficientCount], residual[..coefficientCount], -transformShift, rotate); + } + } + + /// + /// Gets whether a non-transformed residual block uses the HEVC Range Extensions coefficient rotation. + /// + /// Whether the sequence enables transform-skip rotation. + /// Whether the transform unit belongs to an intra-predicted coding unit. + /// The transform-block width. + /// when the complete coefficient order is reversed; otherwise, . + public static bool IsNonTransformedResidualRotated(bool transformSkipRotationEnabled, bool isIntraPredicted, int width) + => transformSkipRotationEnabled && isIntraPredicted && width == 4; + + /// + /// Gets the implicit residual differential mode selected by an intra-prediction direction. + /// + /// The resolved luma or chroma intra-prediction mode. + /// Whether the 4:2:2 chroma intra-angle remapping applies. + /// The residual differential mode selected by the prediction direction. + public static HevcResidualDpcmMode GetImplicitResidualDpcmMode(int intraPredictionMode, bool remapChroma422) + { + int predictionMode = remapChroma422 ? Chroma422IntraAngleMap[intraPredictionMode] : intraPredictionMode; + return predictionMode switch + { + HorizontalIntraPredictionMode => HevcResidualDpcmMode.Horizontal, + VerticalIntraPredictionMode => HevcResidualDpcmMode.Vertical, + _ => HevcResidualDpcmMode.None, + }; + } + + /// + /// Applies inverse residual differential pulse-code modulation to one packed residual block. + /// + /// The residual block in packed raster order. + /// The residual-block width. + /// The residual-block height. + /// The differential accumulation direction. + public static void ApplyResidualDpcm(Span residual, int width, int height, HevcResidualDpcmMode mode) + { + if (mode == HevcResidualDpcmMode.Vertical) + { + ApplyVerticalResidualDpcm(residual, width, height); + } + else if (mode == HevcResidualDpcmMode.Horizontal) + { + ApplyHorizontalResidualDpcm(residual, width, height); + } + } + + /// + /// Applies one transform-skip normalization operator to a complete coefficient block. + /// + /// The signed shift operator selected before entering the hot loop. + /// The dequantized coefficients in raster order. + /// The destination residual block in packed raster order. + /// The nonnegative shift magnitude. + /// Whether the complete coefficient order is reversed. + private static void ApplyTransformSkip(ReadOnlySpan coefficients, Span residual, int shift, bool rotate) + where TOperator : struct, ITransformSkipOperator + { + ref int sourceBase = ref MemoryMarshal.GetReference(coefficients); + ref int destinationBase = ref MemoryMarshal.GetReference(residual); + int count = coefficients.Length; + int index = 0; + + if (Vector512.IsHardwareAccelerated) + { + for (; index <= count - Vector512.Count; index += Vector512.Count) + { + Vector512 values = Load512(ref sourceBase, count, index, rotate); + TOperator.Invoke(values, shift).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector256.IsHardwareAccelerated) + { + for (; index <= count - Vector256.Count; index += Vector256.Count) + { + Vector256 values = Load256(ref sourceBase, count, index, rotate); + TOperator.Invoke(values, shift).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector128.IsHardwareAccelerated) + { + for (; index <= count - Vector128.Count; index += Vector128.Count) + { + Vector128 values = Load128(ref sourceBase, count, index, rotate); + TOperator.Invoke(values, shift).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + for (; index < count; index++) + { + int sourceIndex = rotate ? count - 1 - index : index; + Unsafe.Add(ref destinationBase, index) = TOperator.Invoke(Unsafe.Add(ref sourceBase, sourceIndex), shift); + } + } + + /// + /// Copies one coefficient block while reversing its complete raster order. + /// + /// The source coefficient block. + /// The destination residual block. + private static void CopyReversed(ReadOnlySpan source, Span destination) + { + ref int sourceBase = ref MemoryMarshal.GetReference(source); + ref int destinationBase = ref MemoryMarshal.GetReference(destination); + int count = source.Length; + int index = 0; + + if (Vector512.IsHardwareAccelerated) + { + for (; index <= count - Vector512.Count; index += Vector512.Count) + { + Load512(ref sourceBase, count, index, true).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector256.IsHardwareAccelerated) + { + for (; index <= count - Vector256.Count; index += Vector256.Count) + { + Load256(ref sourceBase, count, index, true).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector128.IsHardwareAccelerated) + { + for (; index <= count - Vector128.Count; index += Vector128.Count) + { + Load128(ref sourceBase, count, index, true).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + for (; index < count; index++) + { + Unsafe.Add(ref destinationBase, index) = Unsafe.Add(ref sourceBase, count - 1 - index); + } + } + + /// + /// Loads and optionally reverses sixteen source coefficients. + /// + /// The first source coefficient. + /// The complete coefficient count. + /// The destination coefficient index. + /// Whether the complete coefficient order is reversed. + /// The source coefficients in destination order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Load512(ref int source, int count, int index, bool rotate) + { + if (!rotate) + { + return Vector512.LoadUnsafe(ref source, (nuint)index); + } + + Vector512 values = Vector512.LoadUnsafe(ref source, (nuint)(count - index - Vector512.Count)); + return Vector512.Shuffle(values, Vector512.Create(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); + } + + /// + /// Loads and optionally reverses eight source coefficients. + /// + /// The first source coefficient. + /// The complete coefficient count. + /// The destination coefficient index. + /// Whether the complete coefficient order is reversed. + /// The source coefficients in destination order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Load256(ref int source, int count, int index, bool rotate) + { + if (!rotate) + { + return Vector256.LoadUnsafe(ref source, (nuint)index); + } + + Vector256 values = Vector256.LoadUnsafe(ref source, (nuint)(count - index - Vector256.Count)); + return Vector256.Shuffle(values, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0)); + } + + /// + /// Loads and optionally reverses four source coefficients. + /// + /// The first source coefficient. + /// The complete coefficient count. + /// The destination coefficient index. + /// Whether the complete coefficient order is reversed. + /// The source coefficients in destination order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Load128(ref int source, int count, int index, bool rotate) + { + if (!rotate) + { + return Vector128.LoadUnsafe(ref source, (nuint)index); + } + + Vector128 values = Vector128.LoadUnsafe(ref source, (nuint)(count - index - Vector128.Count)); + return Vector128.Shuffle(values, Vector128.Create(3, 2, 1, 0)); + } + + /// + /// Accumulates residual differences from top to bottom while processing independent columns in SIMD lanes. + /// + /// The residual block in packed raster order. + /// The residual-block width. + /// The residual-block height. + private static void ApplyVerticalResidualDpcm(Span residual, int width, int height) + { + ref int residualBase = ref MemoryMarshal.GetReference(residual); + int x = 0; + if (Vector512.IsHardwareAccelerated) + { + Vector512 minimum = Vector512.Create(ResidualMinimum); + Vector512 maximum = Vector512.Create(ResidualMaximum); + for (; x <= width - Vector512.Count; x += Vector512.Count) + { + Vector512 accumulator = Vector512.LoadUnsafe(ref residualBase, (nuint)x); + for (int y = 1; y < height; y++) + { + int index = (y * width) + x; + accumulator += Vector512.LoadUnsafe(ref residualBase, (nuint)index); + Vector512.Clamp(accumulator, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)index); + } + } + } + + if (Vector256.IsHardwareAccelerated) + { + Vector256 minimum = Vector256.Create(ResidualMinimum); + Vector256 maximum = Vector256.Create(ResidualMaximum); + for (; x <= width - Vector256.Count; x += Vector256.Count) + { + Vector256 accumulator = Vector256.LoadUnsafe(ref residualBase, (nuint)x); + for (int y = 1; y < height; y++) + { + int index = (y * width) + x; + accumulator += Vector256.LoadUnsafe(ref residualBase, (nuint)index); + Vector256.Clamp(accumulator, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)index); + } + } + } + + if (Vector128.IsHardwareAccelerated) + { + Vector128 minimum = Vector128.Create(ResidualMinimum); + Vector128 maximum = Vector128.Create(ResidualMaximum); + for (; x <= width - Vector128.Count; x += Vector128.Count) + { + Vector128 accumulator = Vector128.LoadUnsafe(ref residualBase, (nuint)x); + for (int y = 1; y < height; y++) + { + int index = (y * width) + x; + accumulator += Vector128.LoadUnsafe(ref residualBase, (nuint)index); + Vector128.Clamp(accumulator, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)index); + } + } + } + + for (; x < width; x++) + { + int accumulator = Unsafe.Add(ref residualBase, x); + for (int y = 1; y < height; y++) + { + int index = (y * width) + x; + accumulator += Unsafe.Add(ref residualBase, index); + Unsafe.Add(ref residualBase, index) = Math.Clamp(accumulator, ResidualMinimum, ResidualMaximum); + } + } + } + + /// + /// Accumulates residual differences from left to right using an inclusive SIMD prefix sum for each row. + /// + /// The residual block in packed raster order. + /// The residual-block width. + /// The residual-block height. + private static void ApplyHorizontalResidualDpcm(Span residual, int width, int height) + { + ref int residualBase = ref MemoryMarshal.GetReference(residual); + for (int y = 0; y < height; y++) + { + int rowOffset = y * width; + int x = 0; + int accumulator = 0; + + if (Vector512.IsHardwareAccelerated) + { + Vector512 minimum = Vector512.Create(ResidualMinimum); + Vector512 maximum = Vector512.Create(ResidualMaximum); + for (; x <= width - Vector512.Count; x += Vector512.Count) + { + Vector512 values = Vector512.LoadUnsafe(ref residualBase, (nuint)(rowOffset + x)); + values = PrefixSum(values) + Vector512.Create(accumulator); + accumulator = values.GetElement(Vector512.Count - 1); + Vector512.Clamp(values, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)(rowOffset + x)); + } + } + + if (Vector256.IsHardwareAccelerated) + { + Vector256 minimum = Vector256.Create(ResidualMinimum); + Vector256 maximum = Vector256.Create(ResidualMaximum); + for (; x <= width - Vector256.Count; x += Vector256.Count) + { + Vector256 values = Vector256.LoadUnsafe(ref residualBase, (nuint)(rowOffset + x)); + values = PrefixSum(values) + Vector256.Create(accumulator); + accumulator = values.GetElement(Vector256.Count - 1); + Vector256.Clamp(values, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)(rowOffset + x)); + } + } + + if (Vector128.IsHardwareAccelerated) + { + Vector128 minimum = Vector128.Create(ResidualMinimum); + Vector128 maximum = Vector128.Create(ResidualMaximum); + for (; x <= width - Vector128.Count; x += Vector128.Count) + { + Vector128 values = Vector128.LoadUnsafe(ref residualBase, (nuint)(rowOffset + x)); + values = PrefixSum(values) + Vector128.Create(accumulator); + accumulator = values.GetElement(Vector128.Count - 1); + Vector128.Clamp(values, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)(rowOffset + x)); + } + } + + for (; x < width; x++) + { + int index = rowOffset + x; + accumulator += Unsafe.Add(ref residualBase, index); + Unsafe.Add(ref residualBase, index) = x == 0 ? accumulator : Math.Clamp(accumulator, ResidualMinimum, ResidualMaximum); + } + } + } + + /// + /// Computes an inclusive prefix sum across sixteen signed lanes. + /// + /// The residual differences. + /// The accumulated residuals. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 PrefixSum(Vector512 values) + { + values += Vector512.Shuffle(values, Vector512.Create(16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); + values += Vector512.Shuffle(values, Vector512.Create(16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)); + values += Vector512.Shuffle(values, Vector512.Create(16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)); + return values + Vector512.Shuffle(values, Vector512.Create(16, 16, 16, 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7)); + } + + /// + /// Computes an inclusive prefix sum across eight signed lanes. + /// + /// The residual differences. + /// The accumulated residuals. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PrefixSum(Vector256 values) + { + values += Vector256.Shuffle(values, Vector256.Create(8, 0, 1, 2, 3, 4, 5, 6)); + values += Vector256.Shuffle(values, Vector256.Create(8, 8, 0, 1, 2, 3, 4, 5)); + return values + Vector256.Shuffle(values, Vector256.Create(8, 8, 8, 8, 0, 1, 2, 3)); + } + + /// + /// Computes an inclusive prefix sum across four signed lanes. + /// + /// The residual differences. + /// The accumulated residuals. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PrefixSum(Vector128 values) + { + values += Vector128.Shuffle(values, Vector128.Create(4, 0, 1, 2)); + return values + Vector128.Shuffle(values, Vector128.Create(4, 4, 0, 1)); + } + + /// + /// Applies the rounded right shift used by ordinary transform-skip reconstruction. + /// + private readonly struct RightShiftTransformSkipOperator : ITransformSkipOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 values, int shift) + => shift == 0 ? values : (values + Vector512.Create(1 << (shift - 1))) >> shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 values, int shift) + => shift == 0 ? values : (values + Vector256.Create(1 << (shift - 1))) >> shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 values, int shift) + => shift == 0 ? values : (values + Vector128.Create(1 << (shift - 1))) >> shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(int value, int shift) => shift == 0 ? value : (value + (1 << (shift - 1))) >> shift; + } + + /// + /// Applies the exact left shift used by high-bit-depth transform-skip reconstruction. + /// + private readonly struct LeftShiftTransformSkipOperator : ITransformSkipOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 values, int shift) => values << shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 values, int shift) => values << shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 values, int shift) => values << shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(int value, int shift) => value << shift; + } +} diff --git a/tests/ImageSharp.Benchmarks/Codecs/Heif/HevcResidualReconstructionBenchmarks.cs b/tests/ImageSharp.Benchmarks/Codecs/Heif/HevcResidualReconstructionBenchmarks.cs new file mode 100644 index 000000000..664b44b67 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Codecs/Heif/HevcResidualReconstructionBenchmarks.cs @@ -0,0 +1,91 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Formats.Heif.Hevc; + +namespace SixLabors.ImageSharp.Benchmarks.Codecs.Heif; + +/// +/// Measures complete coded-frame traversal through HEVC transform-skip and residual differential reconstruction. +/// +[MemoryDiagnoser(displayGenColumns: false)] +public class HevcResidualReconstructionBenchmarks +{ + /// + /// The coded frame width, which is an exact multiple of the maximum transform-block side. + /// + private const int Width = 1920; + + /// + /// The coded frame height including the final padded coding-tree row for a 1080-line presentation. + /// + private const int Height = 1088; + + /// + /// The benchmark transform-block side in samples. + /// + private const int BlockSize = 32; + + /// + /// The deterministic dequantized coefficients reused by each benchmark block. + /// + private readonly int[] coefficients = new int[BlockSize * BlockSize]; + + /// + /// The reusable packed residual block. + /// + private readonly int[] residual = new int[BlockSize * BlockSize]; + + /// + /// Populates a dense, deterministic twelve-bit transform-skip workload outside the measured frame traversal. + /// + [GlobalSetup] + public void Setup() + { + for (int i = 0; i < this.coefficients.Length; i++) + { + this.coefficients[i] = (((i * 104729) + 4099) & 8191) - 4096; + } + } + + /// + /// Measures frame-wide transform-skip normalization. + /// + /// The final residual, keeping the block output observable. + [Benchmark(Baseline = true)] + public int TransformSkipFrame() => this.ReconstructFrame(HevcResidualDpcmMode.None); + + /// + /// Measures frame-wide transform-skip normalization followed by horizontal residual differential reconstruction. + /// + /// The final residual, keeping the block output observable. + [Benchmark] + public int HorizontalResidualDpcmFrame() => this.ReconstructFrame(HevcResidualDpcmMode.Horizontal); + + /// + /// Measures frame-wide transform-skip normalization followed by vertical residual differential reconstruction. + /// + /// The final residual, keeping the block output observable. + [Benchmark] + public int VerticalResidualDpcmFrame() => this.ReconstructFrame(HevcResidualDpcmMode.Vertical); + + /// + /// Reconstructs every maximum-size transform block in the coded benchmark frame. + /// + /// The residual differential mode applied after transform-skip normalization. + /// The final reconstructed residual. + private int ReconstructFrame(HevcResidualDpcmMode mode) + { + for (int y = 0; y < Height; y += BlockSize) + { + for (int x = 0; x < Width; x += BlockSize) + { + HevcResidualReconstructor.ApplyTransformSkip(this.coefficients, this.residual, BlockSize, BlockSize, 12, 18, 5, true, false); + HevcResidualReconstructor.ApplyResidualDpcm(this.residual, BlockSize, BlockSize, mode); + } + } + + return this.residual[^1]; + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcResidualReconstructorTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcResidualReconstructorTests.cs new file mode 100644 index 000000000..2d1f91a53 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcResidualReconstructorTests.cs @@ -0,0 +1,268 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Hevc; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Hevc; + +/// +/// Verifies HEVC transform-skip, transquant-bypass, rotation, and residual differential reconstruction. +/// +[Trait("Format", "Heic")] +public class HevcResidualReconstructorTests +{ + /// + /// Verifies that lossless transquant bypass preserves or completely reverses coefficient order. + /// + /// Whether the coefficient order is reversed. + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CopyBypassedPreservesOrRotatesCoefficientOrder(bool rotate) + { + int[] coefficients = new int[1024]; + int[] actual = new int[coefficients.Length]; + int[] expected = new int[coefficients.Length]; + for (int i = 0; i < coefficients.Length; i++) + { + coefficients[i] = (i * 17) - 8000; + } + + for (int i = 0; i < coefficients.Length; i++) + { + expected[i] = rotate ? coefficients[coefficients.Length - 1 - i] : coefficients[i]; + } + + HevcResidualReconstructor.CopyBypassed(coefficients, actual, rotate); + + int mismatch = expected.AsSpan().SequenceEqual(actual) ? -1 : FindFirstMismatch(expected, actual); + Assert.True(mismatch < 0, mismatch < 0 ? string.Empty : $"Mismatch at {mismatch}: expected {expected[mismatch]}, actual {actual[mismatch]}."); + } + + /// + /// Compares SIMD transform-skip reconstruction with a scalar oracle across transform sizes and signed shift directions. + /// + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component precision. + /// The transform dynamic range excluding its sign bit. + /// The base-two logarithm of the equivalent square transform size. + /// Whether extended transform-skip precision applies. + /// Whether the complete coefficient order is reversed. + [Theory] + [InlineData(4, 4, 8, 15, 2, false, true)] + [InlineData(4, 8, 8, 15, 3, false, false)] + [InlineData(8, 4, 10, 15, 2, false, false)] + [InlineData(8, 8, 10, 15, 3, false, false)] + [InlineData(16, 16, 12, 15, 4, false, false)] + [InlineData(16, 16, 12, 15, 4, true, false)] + [InlineData(32, 32, 12, 18, 5, false, false)] + public void TransformSkipMatchesScalarOracle( + int width, + int height, + int bitDepth, + int maxTransformDynamicRange, + int equivalentLog2TransformSize, + bool extendedPrecisionProcessingEnabled, + bool rotate) + { + int coefficientCount = width * height; + int[] coefficients = new int[coefficientCount]; + int[] actual = new int[coefficientCount]; + int[] expected = new int[coefficientCount]; + for (int i = 0; i < coefficientCount; i++) + { + coefficients[i] = (((i * 7919) + (width * 257)) & 65535) - 32768; + } + + ApplyTransformSkipScalar( + coefficients, + expected, + bitDepth, + maxTransformDynamicRange, + equivalentLog2TransformSize, + extendedPrecisionProcessingEnabled, + rotate); + + HevcResidualReconstructor.ApplyTransformSkip( + coefficients, + actual, + width, + height, + bitDepth, + maxTransformDynamicRange, + equivalentLog2TransformSize, + extendedPrecisionProcessingEnabled, + rotate); + + Assert.True(expected.AsSpan().SequenceEqual(actual)); + } + + /// + /// Compares SIMD residual differential reconstruction with the sequential normative recurrence. + /// + /// The square residual-block side. + /// The numeric differential accumulation direction. + [Theory] + [InlineData(4, 1)] + [InlineData(4, 2)] + [InlineData(8, 1)] + [InlineData(8, 2)] + [InlineData(16, 1)] + [InlineData(16, 2)] + [InlineData(32, 1)] + [InlineData(32, 2)] + public void ResidualDpcmMatchesScalarOracle(int size, int modeValue) + { + HevcResidualDpcmMode mode = (HevcResidualDpcmMode)modeValue; + int[] actual = new int[size * size]; + for (int i = 0; i < actual.Length; i++) + { + actual[i] = (((i * 104729) + (size * 4099)) & 8191) - 4096; + } + + int[] expected = (int[])actual.Clone(); + ApplyResidualDpcmScalar(expected, size, size, mode); + + HevcResidualReconstructor.ApplyResidualDpcm(actual, size, size, mode); + + Assert.True(expected.AsSpan().SequenceEqual(actual)); + } + + /// + /// Verifies signed residual clipping without clipping the thirty-two-bit recurrence accumulator. + /// + /// The numeric differential accumulation direction. + [Theory] + [InlineData(1)] + [InlineData(2)] + public void ResidualDpcmClipsStoredSamples(int modeValue) + { + HevcResidualDpcmMode mode = (HevcResidualDpcmMode)modeValue; + int[] actual = new int[32 * 32]; + actual.AsSpan().Fill(3000); + int[] expected = (int[])actual.Clone(); + ApplyResidualDpcmScalar(expected, 32, 32, mode); + + HevcResidualReconstructor.ApplyResidualDpcm(actual, 32, 32, mode); + + Assert.True(expected.AsSpan().SequenceEqual(actual)); + Assert.Contains(short.MaxValue, actual); + } + + /// + /// Verifies the Range Extensions rotation constraint for non-transformed intra blocks. + /// + [Fact] + public void RotationRequiresEnabledFourWideIntraBlock() + { + Assert.True(HevcResidualReconstructor.IsNonTransformedResidualRotated(true, true, 4)); + Assert.False(HevcResidualReconstructor.IsNonTransformedResidualRotated(false, true, 4)); + Assert.False(HevcResidualReconstructor.IsNonTransformedResidualRotated(true, false, 4)); + Assert.False(HevcResidualReconstructor.IsNonTransformedResidualRotated(true, true, 8)); + } + + /// + /// Verifies implicit residual differential mode selection, including 4:2:2 chroma angle remapping. + /// + [Fact] + public void ImplicitResidualDpcmFollowsPredictionDirection() + { + Assert.Equal(HevcResidualDpcmMode.Horizontal, HevcResidualReconstructor.GetImplicitResidualDpcmMode(10, false)); + Assert.Equal(HevcResidualDpcmMode.Vertical, HevcResidualReconstructor.GetImplicitResidualDpcmMode(26, false)); + Assert.Equal(HevcResidualDpcmMode.None, HevcResidualReconstructor.GetImplicitResidualDpcmMode(18, false)); + Assert.Equal(HevcResidualDpcmMode.Horizontal, HevcResidualReconstructor.GetImplicitResidualDpcmMode(10, true)); + Assert.Equal(HevcResidualDpcmMode.Vertical, HevcResidualReconstructor.GetImplicitResidualDpcmMode(26, true)); + } + + /// + /// Applies the normative transform-skip normalization as a scalar test oracle. + /// + /// The dequantized coefficients. + /// The destination residual block. + /// The reconstructed component precision. + /// The transform dynamic range excluding its sign bit. + /// The base-two logarithm of the equivalent square transform size. + /// Whether extended transform-skip precision applies. + /// Whether the complete coefficient order is reversed. + private static void ApplyTransformSkipScalar( + ReadOnlySpan coefficients, + Span residual, + int bitDepth, + int maxTransformDynamicRange, + int equivalentLog2TransformSize, + bool extendedPrecisionProcessingEnabled, + bool rotate) + { + int shift = maxTransformDynamicRange - bitDepth - equivalentLog2TransformSize; + if (extendedPrecisionProcessingEnabled) + { + shift = Math.Max(0, shift); + } + + for (int i = 0; i < coefficients.Length; i++) + { + int value = coefficients[rotate ? coefficients.Length - 1 - i : i]; + residual[i] = shift > 0 + ? (value + (1 << (shift - 1))) >> shift + : value << -shift; + } + } + + /// + /// Applies the normative inverse residual differential recurrence as a scalar test oracle. + /// + /// The residual block in packed raster order. + /// The residual-block width. + /// The residual-block height. + /// The differential accumulation direction. + private static void ApplyResidualDpcmScalar(Span residual, int width, int height, HevcResidualDpcmMode mode) + { + if (mode == HevcResidualDpcmMode.Vertical) + { + for (int x = 0; x < width; x++) + { + int accumulator = residual[x]; + for (int y = 1; y < height; y++) + { + int index = (y * width) + x; + accumulator += residual[index]; + residual[index] = Math.Clamp(accumulator, short.MinValue, short.MaxValue); + } + } + } + else if (mode == HevcResidualDpcmMode.Horizontal) + { + for (int y = 0; y < height; y++) + { + int rowOffset = y * width; + int accumulator = residual[rowOffset]; + for (int x = 1; x < width; x++) + { + int index = rowOffset + x; + accumulator += residual[index]; + residual[index] = Math.Clamp(accumulator, short.MinValue, short.MaxValue); + } + } + } + } + + /// + /// Finds the first unequal element in two equally sized test buffers. + /// + /// The expected values. + /// The actual values. + /// The first unequal index. + private static int FindFirstMismatch(ReadOnlySpan expected, ReadOnlySpan actual) + { + for (int i = 0; i < expected.Length; i++) + { + if (expected[i] != actual[i]) + { + return i; + } + } + + return -1; + } +}