Implement SIMD-first HEVC residual reconstruction

HEIF_IMPLEMENTATION_PLAN.md

@@ -474,6 +474,8 @@ Implement and verify in dependency order:
     - [x] Implement the immutable effective-QP value used by reconstruction, including independent luma/chroma bit-depth offsets, the normative 4:2:0 mapping plateaus, the 4:2:2/4:4:4 saturation rule, and combined picture/slice/coding-unit chroma offsets.
     - [ ] Select each transform unit's coding-unit luma QP and chroma-adjustment-list entry, then pass the derived component QP into inverse quantization.
   - [ ] Implement transform skip, coefficient rotation, implicit and explicit residual DPCM, transquant bypass, and lossless reconstruction.
+    - [x] Implement allocation-free SIMD-first transform-skip normalization, complete-block coefficient rotation, transquant-bypass copying, implicit intra-direction selection, and horizontal/vertical inverse residual DPCM with a scalar fallback and signed residual clipping.
+    - [ ] Decode explicit inter residual-DPCM modes and connect bypass, transform skip, residual DPCM, prediction addition, and lossless reconstruction through transform-unit traversal.
   - [ ] Connect coefficient decoding, inverse quantization, transform selection, reusable scratch, and add/clip to transform-unit traversal.
 - [ ] Deblocking and sample-adaptive offset for every signaled luma/chroma and bit-depth path.
 - [ ] Tiles, wavefront entry points, dependent slices, and all other parallelization syntax permitted by the exposed still-image profiles.
@@ -572,6 +574,7 @@ Tasks:
   - [x] Add a permanent frame-wide HEVC reference-preparation benchmark. On .NET 10, complete and partially substituted borders measured 465.3 and 566.7 microseconds per 2,040-block padded 1920x1088 frame, compared with forced-scalar timings of 477.5 and 608.0 microseconds, with zero managed allocations.
   - [x] Add a permanent frame-wide HEVC inverse-transform benchmark. On .NET 10, dense-coefficient 32x32 twelve-bit inverse DCT, transposition, and add/clip measured 3.643 milliseconds per padded 1920x1088 frame, compared with 45.23 milliseconds with hardware intrinsics disabled: 12.4 times faster with zero managed allocations.
   - [x] Add a permanent frame-wide HEVC inverse-quantization benchmark. On .NET 10, dense 32x32 twelve-bit flat and scaling-list paths measured 69.98 and 238.4 microseconds per padded 1920x1088 frame, compared with forced-scalar timings of 1.469 and 1.609 milliseconds: 21.0 and 6.7 times faster with zero managed allocations. Pre-expanding the scaling matrices once reduced the SIMD scaling-list path from 4.974 milliseconds to 238.4 microseconds.
+  - [x] Add a permanent frame-wide HEVC residual-reconstruction benchmark. On .NET 10, dense 32x32 twelve-bit transform skip, horizontal RDPCM, and vertical RDPCM measured 79.64, 275.61, and 110.81 microseconds per padded 1920x1088 frame, compared with forced-scalar timings of 571.5 microseconds, 1.512 milliseconds, and 1.257 milliseconds: 7.2, 5.5, and 11.3 times faster with zero managed allocations.
 - [ ] Implement vector paths only for confirmed hot loops, using existing `Vector128`, `Vector256`, and `Vector512` helper and dispatch patterns where supported.
 - [ ] Prioritize shared color conversion and pixel packing, chroma upsampling, inverse-transform add-and-clip, intra predictors, HEVC deblock/SAO, AV1 loop filter/CDEF/restoration, and contiguous grid copies.
 - [ ] Benchmark the complete decode color pipeline on representative 8/10/12-bit AVIF and HEIC images with and without embedded ICC profiles. Report absolute end-to-end timings and allocations in addition to the isolated YUV/CICP and ICC stage costs.

src/ImageSharp/Formats/Heif/Hevc/HevcResidualDpcmMode.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <summary>
+/// Identifies the differential pulse-code modulation applied to an HEVC residual block.
+/// </summary>
+internal enum HevcResidualDpcmMode : byte
+{
+    /// <summary>
+    /// No residual differential pulse-code modulation is applied.
+    /// </summary>
+    None = 0,
+
+    /// <summary>
+    /// Residual differences accumulate from left to right within each row.
+    /// </summary>
+    Horizontal = 1,
+
+    /// <summary>
+    /// Residual differences accumulate from top to bottom within each column.
+    /// </summary>
+    Vertical = 2,
+}

src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs

@@ -0,0 +1,555 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <summary>
+/// Reconstructs HEVC transform-skipped, bypassed, and differential residual blocks.
+/// </summary>
+internal static class HevcResidualReconstructor
+{
+    /// <summary>
+    /// The horizontal intra-prediction mode defined by H.265.
+    /// </summary>
+    private const int HorizontalIntraPredictionMode = 10;
+
+    /// <summary>
+    /// The vertical intra-prediction mode defined by H.265.
+    /// </summary>
+    private const int VerticalIntraPredictionMode = 26;
+
+    /// <summary>
+    /// The minimum residual sample represented by the decoder reconstruction pipeline.
+    /// </summary>
+    private const int ResidualMinimum = short.MinValue;
+
+    /// <summary>
+    /// The maximum residual sample represented by the decoder reconstruction pipeline.
+    /// </summary>
+    private const int ResidualMaximum = short.MaxValue;
+
+    /// <summary>
+    /// Defines a closed transform-skip normalization operator for every SIMD width and the scalar tail.
+    /// </summary>
+    private interface ITransformSkipOperator
+    {
+        /// <summary>
+        /// Normalizes sixteen transform-skipped coefficients.
+        /// </summary>
+        /// <param name="values">The dequantized coefficients.</param>
+        /// <param name="shift">The nonnegative shift magnitude.</param>
+        /// <returns>The reconstructed residuals.</returns>
+        static abstract Vector512<int> Invoke(Vector512<int> values, int shift);
+
+        /// <summary>
+        /// Normalizes eight transform-skipped coefficients.
+        /// </summary>
+        /// <param name="values">The dequantized coefficients.</param>
+        /// <param name="shift">The nonnegative shift magnitude.</param>
+        /// <returns>The reconstructed residuals.</returns>
+        static abstract Vector256<int> Invoke(Vector256<int> values, int shift);
+
+        /// <summary>
+        /// Normalizes four transform-skipped coefficients.
+        /// </summary>
+        /// <param name="values">The dequantized coefficients.</param>
+        /// <param name="shift">The nonnegative shift magnitude.</param>
+        /// <returns>The reconstructed residuals.</returns>
+        static abstract Vector128<int> Invoke(Vector128<int> values, int shift);
+
+        /// <summary>
+        /// Normalizes one transform-skipped coefficient.
+        /// </summary>
+        /// <param name="value">The dequantized coefficient.</param>
+        /// <param name="shift">The nonnegative shift magnitude.</param>
+        /// <returns>The reconstructed residual.</returns>
+        static abstract int Invoke(int value, int shift);
+    }
+
+    /// <summary>
+    /// Gets the 4:2:2 chroma intra-angle remapping defined by H.265 Table 8-4.
+    /// </summary>
+    private static ReadOnlySpan<byte> Chroma422IntraAngleMap =>
+    [
+        0, 1, 2, 2, 2, 2, 3, 5, 7, 8, 10, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 23, 24, 24, 25, 25, 26, 27, 27, 28, 28, 29, 29, 30, 31,
+    ];
+
+    /// <summary>
+    /// Copies one transquant-bypass coefficient block into residual sample order.
+    /// </summary>
+    /// <param name="coefficients">The decoded coefficients in raster order.</param>
+    /// <param name="residual">The destination residual block in packed raster order.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    public static void CopyBypassed(ReadOnlySpan<int> coefficients, Span<int> residual, bool rotate)
+    {
+        Span<int> destination = residual[..coefficients.Length];
+        if (!rotate)
+        {
+            coefficients.CopyTo(destination);
+            return;
+        }
+
+        CopyReversed(coefficients, destination);
+    }
+
+    /// <summary>
+    /// Reconstructs one transform-skipped residual block from dequantized coefficients.
+    /// </summary>
+    /// <param name="coefficients">The dequantized coefficients in raster order.</param>
+    /// <param name="residual">The destination residual block in packed raster order.</param>
+    /// <param name="width">The transform-block width.</param>
+    /// <param name="height">The transform-block height.</param>
+    /// <param name="bitDepth">The reconstructed component precision.</param>
+    /// <param name="maxTransformDynamicRange">The transform dynamic range excluding its sign bit.</param>
+    /// <param name="equivalentLog2TransformSize">The base-two logarithm of the equivalent square transform size.</param>
+    /// <param name="extendedPrecisionProcessingEnabled">Whether transform-skip precision is extended by the sequence.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    public static void ApplyTransformSkip(
+        ReadOnlySpan<int> coefficients,
+        Span<int> residual,
+        int width,
+        int height,
+        int bitDepth,
+        int maxTransformDynamicRange,
+        int equivalentLog2TransformSize,
+        bool extendedPrecisionProcessingEnabled,
+        bool rotate)
+    {
+        int transformShift = maxTransformDynamicRange - bitDepth - equivalentLog2TransformSize;
+        if (extendedPrecisionProcessingEnabled)
+        {
+            transformShift = Math.Max(0, transformShift);
+        }
+
+        int coefficientCount = width * height;
+        if (transformShift >= 0)
+        {
+            ApplyTransformSkip<RightShiftTransformSkipOperator>(coefficients[..coefficientCount], residual[..coefficientCount], transformShift, rotate);
+        }
+        else
+        {
+            ApplyTransformSkip<LeftShiftTransformSkipOperator>(coefficients[..coefficientCount], residual[..coefficientCount], -transformShift, rotate);
+        }
+    }
+
+    /// <summary>
+    /// Gets whether a non-transformed residual block uses the HEVC Range Extensions coefficient rotation.
+    /// </summary>
+    /// <param name="transformSkipRotationEnabled">Whether the sequence enables transform-skip rotation.</param>
+    /// <param name="isIntraPredicted">Whether the transform unit belongs to an intra-predicted coding unit.</param>
+    /// <param name="width">The transform-block width.</param>
+    /// <returns><see langword="true"/> when the complete coefficient order is reversed; otherwise, <see langword="false"/>.</returns>
+    public static bool IsNonTransformedResidualRotated(bool transformSkipRotationEnabled, bool isIntraPredicted, int width)
+        => transformSkipRotationEnabled && isIntraPredicted && width == 4;
+
+    /// <summary>
+    /// Gets the implicit residual differential mode selected by an intra-prediction direction.
+    /// </summary>
+    /// <param name="intraPredictionMode">The resolved luma or chroma intra-prediction mode.</param>
+    /// <param name="remapChroma422">Whether the 4:2:2 chroma intra-angle remapping applies.</param>
+    /// <returns>The residual differential mode selected by the prediction direction.</returns>
+    public static HevcResidualDpcmMode GetImplicitResidualDpcmMode(int intraPredictionMode, bool remapChroma422)
+    {
+        int predictionMode = remapChroma422 ? Chroma422IntraAngleMap[intraPredictionMode] : intraPredictionMode;
+        return predictionMode switch
+        {
+            HorizontalIntraPredictionMode => HevcResidualDpcmMode.Horizontal,
+            VerticalIntraPredictionMode => HevcResidualDpcmMode.Vertical,
+            _ => HevcResidualDpcmMode.None,
+        };
+    }
+
+    /// <summary>
+    /// Applies inverse residual differential pulse-code modulation to one packed residual block.
+    /// </summary>
+    /// <param name="residual">The residual block in packed raster order.</param>
+    /// <param name="width">The residual-block width.</param>
+    /// <param name="height">The residual-block height.</param>
+    /// <param name="mode">The differential accumulation direction.</param>
+    public static void ApplyResidualDpcm(Span<int> residual, int width, int height, HevcResidualDpcmMode mode)
+    {
+        if (mode == HevcResidualDpcmMode.Vertical)
+        {
+            ApplyVerticalResidualDpcm(residual, width, height);
+        }
+        else if (mode == HevcResidualDpcmMode.Horizontal)
+        {
+            ApplyHorizontalResidualDpcm(residual, width, height);
+        }
+    }
+
+    /// <summary>
+    /// Applies one transform-skip normalization operator to a complete coefficient block.
+    /// </summary>
+    /// <typeparam name="TOperator">The signed shift operator selected before entering the hot loop.</typeparam>
+    /// <param name="coefficients">The dequantized coefficients in raster order.</param>
+    /// <param name="residual">The destination residual block in packed raster order.</param>
+    /// <param name="shift">The nonnegative shift magnitude.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    private static void ApplyTransformSkip<TOperator>(ReadOnlySpan<int> coefficients, Span<int> residual, int shift, bool rotate)
+        where TOperator : struct, ITransformSkipOperator
+    {
+        ref int sourceBase = ref MemoryMarshal.GetReference(coefficients);
+        ref int destinationBase = ref MemoryMarshal.GetReference(residual);
+        int count = coefficients.Length;
+        int index = 0;
+
+        if (Vector512.IsHardwareAccelerated)
+        {
+            for (; index <= count - Vector512<int>.Count; index += Vector512<int>.Count)
+            {
+                Vector512<int> values = Load512(ref sourceBase, count, index, rotate);
+                TOperator.Invoke(values, shift).StoreUnsafe(ref destinationBase, (nuint)index);
+            }
+        }
+
+        if (Vector256.IsHardwareAccelerated)
+        {
+            for (; index <= count - Vector256<int>.Count; index += Vector256<int>.Count)
+            {
+                Vector256<int> values = Load256(ref sourceBase, count, index, rotate);
+                TOperator.Invoke(values, shift).StoreUnsafe(ref destinationBase, (nuint)index);
+            }
+        }
+
+        if (Vector128.IsHardwareAccelerated)
+        {
+            for (; index <= count - Vector128<int>.Count; index += Vector128<int>.Count)
+            {
+                Vector128<int> values = Load128(ref sourceBase, count, index, rotate);
+                TOperator.Invoke(values, shift).StoreUnsafe(ref destinationBase, (nuint)index);
+            }
+        }
+
+        for (; index < count; index++)
+        {
+            int sourceIndex = rotate ? count - 1 - index : index;
+            Unsafe.Add(ref destinationBase, index) = TOperator.Invoke(Unsafe.Add(ref sourceBase, sourceIndex), shift);
+        }
+    }
+
+    /// <summary>
+    /// Copies one coefficient block while reversing its complete raster order.
+    /// </summary>
+    /// <param name="source">The source coefficient block.</param>
+    /// <param name="destination">The destination residual block.</param>
+    private static void CopyReversed(ReadOnlySpan<int> source, Span<int> destination)
+    {
+        ref int sourceBase = ref MemoryMarshal.GetReference(source);
+        ref int destinationBase = ref MemoryMarshal.GetReference(destination);
+        int count = source.Length;
+        int index = 0;
+
+        if (Vector512.IsHardwareAccelerated)
+        {
+            for (; index <= count - Vector512<int>.Count; index += Vector512<int>.Count)
+            {
+                Load512(ref sourceBase, count, index, true).StoreUnsafe(ref destinationBase, (nuint)index);
+            }
+        }
+
+        if (Vector256.IsHardwareAccelerated)
+        {
+            for (; index <= count - Vector256<int>.Count; index += Vector256<int>.Count)
+            {
+                Load256(ref sourceBase, count, index, true).StoreUnsafe(ref destinationBase, (nuint)index);
+            }
+        }
+
+        if (Vector128.IsHardwareAccelerated)
+        {
+            for (; index <= count - Vector128<int>.Count; index += Vector128<int>.Count)
+            {
+                Load128(ref sourceBase, count, index, true).StoreUnsafe(ref destinationBase, (nuint)index);
+            }
+        }
+
+        for (; index < count; index++)
+        {
+            Unsafe.Add(ref destinationBase, index) = Unsafe.Add(ref sourceBase, count - 1 - index);
+        }
+    }
+
+    /// <summary>
+    /// Loads and optionally reverses sixteen source coefficients.
+    /// </summary>
+    /// <param name="source">The first source coefficient.</param>
+    /// <param name="count">The complete coefficient count.</param>
+    /// <param name="index">The destination coefficient index.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    /// <returns>The source coefficients in destination order.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static Vector512<int> Load512(ref int source, int count, int index, bool rotate)
+    {
+        if (!rotate)
+        {
+            return Vector512.LoadUnsafe(ref source, (nuint)index);
+        }
+
+        Vector512<int> values = Vector512.LoadUnsafe(ref source, (nuint)(count - index - Vector512<int>.Count));
+        return Vector512.Shuffle(values, Vector512.Create(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
+    }
+
+    /// <summary>
+    /// Loads and optionally reverses eight source coefficients.
+    /// </summary>
+    /// <param name="source">The first source coefficient.</param>
+    /// <param name="count">The complete coefficient count.</param>
+    /// <param name="index">The destination coefficient index.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    /// <returns>The source coefficients in destination order.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static Vector256<int> Load256(ref int source, int count, int index, bool rotate)
+    {
+        if (!rotate)
+        {
+            return Vector256.LoadUnsafe(ref source, (nuint)index);
+        }
+
+        Vector256<int> values = Vector256.LoadUnsafe(ref source, (nuint)(count - index - Vector256<int>.Count));
+        return Vector256.Shuffle(values, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0));
+    }
+
+    /// <summary>
+    /// Loads and optionally reverses four source coefficients.
+    /// </summary>
+    /// <param name="source">The first source coefficient.</param>
+    /// <param name="count">The complete coefficient count.</param>
+    /// <param name="index">The destination coefficient index.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    /// <returns>The source coefficients in destination order.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static Vector128<int> Load128(ref int source, int count, int index, bool rotate)
+    {
+        if (!rotate)
+        {
+            return Vector128.LoadUnsafe(ref source, (nuint)index);
+        }
+
+        Vector128<int> values = Vector128.LoadUnsafe(ref source, (nuint)(count - index - Vector128<int>.Count));
+        return Vector128.Shuffle(values, Vector128.Create(3, 2, 1, 0));
+    }
+
+    /// <summary>
+    /// Accumulates residual differences from top to bottom while processing independent columns in SIMD lanes.
+    /// </summary>
+    /// <param name="residual">The residual block in packed raster order.</param>
+    /// <param name="width">The residual-block width.</param>
+    /// <param name="height">The residual-block height.</param>
+    private static void ApplyVerticalResidualDpcm(Span<int> residual, int width, int height)
+    {
+        ref int residualBase = ref MemoryMarshal.GetReference(residual);
+        int x = 0;
+        if (Vector512.IsHardwareAccelerated)
+        {
+            Vector512<int> minimum = Vector512.Create(ResidualMinimum);
+            Vector512<int> maximum = Vector512.Create(ResidualMaximum);
+            for (; x <= width - Vector512<int>.Count; x += Vector512<int>.Count)
+            {
+                Vector512<int> accumulator = Vector512.LoadUnsafe(ref residualBase, (nuint)x);
+                for (int y = 1; y < height; y++)
+                {
+                    int index = (y * width) + x;
+                    accumulator += Vector512.LoadUnsafe(ref residualBase, (nuint)index);
+                    Vector512.Clamp(accumulator, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)index);
+                }
+            }
+        }
+
+        if (Vector256.IsHardwareAccelerated)
+        {
+            Vector256<int> minimum = Vector256.Create(ResidualMinimum);
+            Vector256<int> maximum = Vector256.Create(ResidualMaximum);
+            for (; x <= width - Vector256<int>.Count; x += Vector256<int>.Count)
+            {
+                Vector256<int> accumulator = Vector256.LoadUnsafe(ref residualBase, (nuint)x);
+                for (int y = 1; y < height; y++)
+                {
+                    int index = (y * width) + x;
+                    accumulator += Vector256.LoadUnsafe(ref residualBase, (nuint)index);
+                    Vector256.Clamp(accumulator, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)index);
+                }
+            }
+        }
+
+        if (Vector128.IsHardwareAccelerated)
+        {
+            Vector128<int> minimum = Vector128.Create(ResidualMinimum);
+            Vector128<int> maximum = Vector128.Create(ResidualMaximum);
+            for (; x <= width - Vector128<int>.Count; x += Vector128<int>.Count)
+            {
+                Vector128<int> accumulator = Vector128.LoadUnsafe(ref residualBase, (nuint)x);
+                for (int y = 1; y < height; y++)
+                {
+                    int index = (y * width) + x;
+                    accumulator += Vector128.LoadUnsafe(ref residualBase, (nuint)index);
+                    Vector128.Clamp(accumulator, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)index);
+                }
+            }
+        }
+
+        for (; x < width; x++)
+        {
+            int accumulator = Unsafe.Add(ref residualBase, x);
+            for (int y = 1; y < height; y++)
+            {
+                int index = (y * width) + x;
+                accumulator += Unsafe.Add(ref residualBase, index);
+                Unsafe.Add(ref residualBase, index) = Math.Clamp(accumulator, ResidualMinimum, ResidualMaximum);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Accumulates residual differences from left to right using an inclusive SIMD prefix sum for each row.
+    /// </summary>
+    /// <param name="residual">The residual block in packed raster order.</param>
+    /// <param name="width">The residual-block width.</param>
+    /// <param name="height">The residual-block height.</param>
+    private static void ApplyHorizontalResidualDpcm(Span<int> residual, int width, int height)
+    {
+        ref int residualBase = ref MemoryMarshal.GetReference(residual);
+        for (int y = 0; y < height; y++)
+        {
+            int rowOffset = y * width;
+            int x = 0;
+            int accumulator = 0;
+
+            if (Vector512.IsHardwareAccelerated)
+            {
+                Vector512<int> minimum = Vector512.Create(ResidualMinimum);
+                Vector512<int> maximum = Vector512.Create(ResidualMaximum);
+                for (; x <= width - Vector512<int>.Count; x += Vector512<int>.Count)
+                {
+                    Vector512<int> values = Vector512.LoadUnsafe(ref residualBase, (nuint)(rowOffset + x));
+                    values = PrefixSum(values) + Vector512.Create(accumulator);
+                    accumulator = values.GetElement(Vector512<int>.Count - 1);
+                    Vector512.Clamp(values, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)(rowOffset + x));
+                }
+            }
+
+            if (Vector256.IsHardwareAccelerated)
+            {
+                Vector256<int> minimum = Vector256.Create(ResidualMinimum);
+                Vector256<int> maximum = Vector256.Create(ResidualMaximum);
+                for (; x <= width - Vector256<int>.Count; x += Vector256<int>.Count)
+                {
+                    Vector256<int> values = Vector256.LoadUnsafe(ref residualBase, (nuint)(rowOffset + x));
+                    values = PrefixSum(values) + Vector256.Create(accumulator);
+                    accumulator = values.GetElement(Vector256<int>.Count - 1);
+                    Vector256.Clamp(values, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)(rowOffset + x));
+                }
+            }
+
+            if (Vector128.IsHardwareAccelerated)
+            {
+                Vector128<int> minimum = Vector128.Create(ResidualMinimum);
+                Vector128<int> maximum = Vector128.Create(ResidualMaximum);
+                for (; x <= width - Vector128<int>.Count; x += Vector128<int>.Count)
+                {
+                    Vector128<int> values = Vector128.LoadUnsafe(ref residualBase, (nuint)(rowOffset + x));
+                    values = PrefixSum(values) + Vector128.Create(accumulator);
+                    accumulator = values.GetElement(Vector128<int>.Count - 1);
+                    Vector128.Clamp(values, minimum, maximum).StoreUnsafe(ref residualBase, (nuint)(rowOffset + x));
+                }
+            }
+
+            for (; x < width; x++)
+            {
+                int index = rowOffset + x;
+                accumulator += Unsafe.Add(ref residualBase, index);
+                Unsafe.Add(ref residualBase, index) = x == 0 ? accumulator : Math.Clamp(accumulator, ResidualMinimum, ResidualMaximum);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Computes an inclusive prefix sum across sixteen signed lanes.
+    /// </summary>
+    /// <param name="values">The residual differences.</param>
+    /// <returns>The accumulated residuals.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static Vector512<int> PrefixSum(Vector512<int> values)
+    {
+        values += Vector512.Shuffle(values, Vector512.Create(16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14));
+        values += Vector512.Shuffle(values, Vector512.Create(16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13));
+        values += Vector512.Shuffle(values, Vector512.Create(16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11));
+        return values + Vector512.Shuffle(values, Vector512.Create(16, 16, 16, 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7));
+    }
+
+    /// <summary>
+    /// Computes an inclusive prefix sum across eight signed lanes.
+    /// </summary>
+    /// <param name="values">The residual differences.</param>
+    /// <returns>The accumulated residuals.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static Vector256<int> PrefixSum(Vector256<int> values)
+    {
+        values += Vector256.Shuffle(values, Vector256.Create(8, 0, 1, 2, 3, 4, 5, 6));
+        values += Vector256.Shuffle(values, Vector256.Create(8, 8, 0, 1, 2, 3, 4, 5));
+        return values + Vector256.Shuffle(values, Vector256.Create(8, 8, 8, 8, 0, 1, 2, 3));
+    }
+
+    /// <summary>
+    /// Computes an inclusive prefix sum across four signed lanes.
+    /// </summary>
+    /// <param name="values">The residual differences.</param>
+    /// <returns>The accumulated residuals.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static Vector128<int> PrefixSum(Vector128<int> values)
+    {
+        values += Vector128.Shuffle(values, Vector128.Create(4, 0, 1, 2));
+        return values + Vector128.Shuffle(values, Vector128.Create(4, 4, 0, 1));
+    }
+
+    /// <summary>
+    /// Applies the rounded right shift used by ordinary transform-skip reconstruction.
+    /// </summary>
+    private readonly struct RightShiftTransformSkipOperator : ITransformSkipOperator
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector512<int> Invoke(Vector512<int> values, int shift)
+            => shift == 0 ? values : (values + Vector512.Create(1 << (shift - 1))) >> shift;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector256<int> Invoke(Vector256<int> values, int shift)
+            => shift == 0 ? values : (values + Vector256.Create(1 << (shift - 1))) >> shift;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> Invoke(Vector128<int> values, int shift)
+            => shift == 0 ? values : (values + Vector128.Create(1 << (shift - 1))) >> shift;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int Invoke(int value, int shift) => shift == 0 ? value : (value + (1 << (shift - 1))) >> shift;
+    }
+
+    /// <summary>
+    /// Applies the exact left shift used by high-bit-depth transform-skip reconstruction.
+    /// </summary>
+    private readonly struct LeftShiftTransformSkipOperator : ITransformSkipOperator
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector512<int> Invoke(Vector512<int> values, int shift) => values << shift;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector256<int> Invoke(Vector256<int> values, int shift) => values << shift;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> Invoke(Vector128<int> values, int shift) => values << shift;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int Invoke(int value, int shift) => value << shift;
+    }
+}

tests/ImageSharp.Benchmarks/Codecs/Heif/HevcResidualReconstructionBenchmarks.cs

@@ -0,0 +1,91 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using BenchmarkDotNet.Attributes;
+using SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+namespace SixLabors.ImageSharp.Benchmarks.Codecs.Heif;
+
+/// <summary>
+/// Measures complete coded-frame traversal through HEVC transform-skip and residual differential reconstruction.
+/// </summary>
+[MemoryDiagnoser(displayGenColumns: false)]
+public class HevcResidualReconstructionBenchmarks
+{
+    /// <summary>
+    /// The coded frame width, which is an exact multiple of the maximum transform-block side.
+    /// </summary>
+    private const int Width = 1920;
+
+    /// <summary>
+    /// The coded frame height including the final padded coding-tree row for a 1080-line presentation.
+    /// </summary>
+    private const int Height = 1088;
+
+    /// <summary>
+    /// The benchmark transform-block side in samples.
+    /// </summary>
+    private const int BlockSize = 32;
+
+    /// <summary>
+    /// The deterministic dequantized coefficients reused by each benchmark block.
+    /// </summary>
+    private readonly int[] coefficients = new int[BlockSize * BlockSize];
+
+    /// <summary>
+    /// The reusable packed residual block.
+    /// </summary>
+    private readonly int[] residual = new int[BlockSize * BlockSize];
+
+    /// <summary>
+    /// Populates a dense, deterministic twelve-bit transform-skip workload outside the measured frame traversal.
+    /// </summary>
+    [GlobalSetup]
+    public void Setup()
+    {
+        for (int i = 0; i < this.coefficients.Length; i++)
+        {
+            this.coefficients[i] = (((i * 104729) + 4099) & 8191) - 4096;
+        }
+    }
+
+    /// <summary>
+    /// Measures frame-wide transform-skip normalization.
+    /// </summary>
+    /// <returns>The final residual, keeping the block output observable.</returns>
+    [Benchmark(Baseline = true)]
+    public int TransformSkipFrame() => this.ReconstructFrame(HevcResidualDpcmMode.None);
+
+    /// <summary>
+    /// Measures frame-wide transform-skip normalization followed by horizontal residual differential reconstruction.
+    /// </summary>
+    /// <returns>The final residual, keeping the block output observable.</returns>
+    [Benchmark]
+    public int HorizontalResidualDpcmFrame() => this.ReconstructFrame(HevcResidualDpcmMode.Horizontal);
+
+    /// <summary>
+    /// Measures frame-wide transform-skip normalization followed by vertical residual differential reconstruction.
+    /// </summary>
+    /// <returns>The final residual, keeping the block output observable.</returns>
+    [Benchmark]
+    public int VerticalResidualDpcmFrame() => this.ReconstructFrame(HevcResidualDpcmMode.Vertical);
+
+    /// <summary>
+    /// Reconstructs every maximum-size transform block in the coded benchmark frame.
+    /// </summary>
+    /// <param name="mode">The residual differential mode applied after transform-skip normalization.</param>
+    /// <returns>The final reconstructed residual.</returns>
+    private int ReconstructFrame(HevcResidualDpcmMode mode)
+    {
+        for (int y = 0; y < Height; y += BlockSize)
+        {
+            for (int x = 0; x < Width; x += BlockSize)
+            {
+                HevcResidualReconstructor.ApplyTransformSkip(this.coefficients, this.residual, BlockSize, BlockSize, 12, 18, 5, true, false);
+                HevcResidualReconstructor.ApplyResidualDpcm(this.residual, BlockSize, BlockSize, mode);
+            }
+        }
+
+        return this.residual[^1];
+    }
+}

tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcResidualReconstructorTests.cs

@@ -0,0 +1,268 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+namespace SixLabors.ImageSharp.Tests.Formats.Heif.Hevc;
+
+/// <summary>
+/// Verifies HEVC transform-skip, transquant-bypass, rotation, and residual differential reconstruction.
+/// </summary>
+[Trait("Format", "Heic")]
+public class HevcResidualReconstructorTests
+{
+    /// <summary>
+    /// Verifies that lossless transquant bypass preserves or completely reverses coefficient order.
+    /// </summary>
+    /// <param name="rotate">Whether the coefficient order is reversed.</param>
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void CopyBypassedPreservesOrRotatesCoefficientOrder(bool rotate)
+    {
+        int[] coefficients = new int[1024];
+        int[] actual = new int[coefficients.Length];
+        int[] expected = new int[coefficients.Length];
+        for (int i = 0; i < coefficients.Length; i++)
+        {
+            coefficients[i] = (i * 17) - 8000;
+        }
+
+        for (int i = 0; i < coefficients.Length; i++)
+        {
+            expected[i] = rotate ? coefficients[coefficients.Length - 1 - i] : coefficients[i];
+        }
+
+        HevcResidualReconstructor.CopyBypassed(coefficients, actual, rotate);
+
+        int mismatch = expected.AsSpan().SequenceEqual(actual) ? -1 : FindFirstMismatch(expected, actual);
+        Assert.True(mismatch < 0, mismatch < 0 ? string.Empty : $"Mismatch at {mismatch}: expected {expected[mismatch]}, actual {actual[mismatch]}.");
+    }
+
+    /// <summary>
+    /// Compares SIMD transform-skip reconstruction with a scalar oracle across transform sizes and signed shift directions.
+    /// </summary>
+    /// <param name="width">The transform-block width.</param>
+    /// <param name="height">The transform-block height.</param>
+    /// <param name="bitDepth">The reconstructed component precision.</param>
+    /// <param name="maxTransformDynamicRange">The transform dynamic range excluding its sign bit.</param>
+    /// <param name="equivalentLog2TransformSize">The base-two logarithm of the equivalent square transform size.</param>
+    /// <param name="extendedPrecisionProcessingEnabled">Whether extended transform-skip precision applies.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    [Theory]
+    [InlineData(4, 4, 8, 15, 2, false, true)]
+    [InlineData(4, 8, 8, 15, 3, false, false)]
+    [InlineData(8, 4, 10, 15, 2, false, false)]
+    [InlineData(8, 8, 10, 15, 3, false, false)]
+    [InlineData(16, 16, 12, 15, 4, false, false)]
+    [InlineData(16, 16, 12, 15, 4, true, false)]
+    [InlineData(32, 32, 12, 18, 5, false, false)]
+    public void TransformSkipMatchesScalarOracle(
+        int width,
+        int height,
+        int bitDepth,
+        int maxTransformDynamicRange,
+        int equivalentLog2TransformSize,
+        bool extendedPrecisionProcessingEnabled,
+        bool rotate)
+    {
+        int coefficientCount = width * height;
+        int[] coefficients = new int[coefficientCount];
+        int[] actual = new int[coefficientCount];
+        int[] expected = new int[coefficientCount];
+        for (int i = 0; i < coefficientCount; i++)
+        {
+            coefficients[i] = (((i * 7919) + (width * 257)) & 65535) - 32768;
+        }
+
+        ApplyTransformSkipScalar(
+            coefficients,
+            expected,
+            bitDepth,
+            maxTransformDynamicRange,
+            equivalentLog2TransformSize,
+            extendedPrecisionProcessingEnabled,
+            rotate);
+
+        HevcResidualReconstructor.ApplyTransformSkip(
+            coefficients,
+            actual,
+            width,
+            height,
+            bitDepth,
+            maxTransformDynamicRange,
+            equivalentLog2TransformSize,
+            extendedPrecisionProcessingEnabled,
+            rotate);
+
+        Assert.True(expected.AsSpan().SequenceEqual(actual));
+    }
+
+    /// <summary>
+    /// Compares SIMD residual differential reconstruction with the sequential normative recurrence.
+    /// </summary>
+    /// <param name="size">The square residual-block side.</param>
+    /// <param name="modeValue">The numeric differential accumulation direction.</param>
+    [Theory]
+    [InlineData(4, 1)]
+    [InlineData(4, 2)]
+    [InlineData(8, 1)]
+    [InlineData(8, 2)]
+    [InlineData(16, 1)]
+    [InlineData(16, 2)]
+    [InlineData(32, 1)]
+    [InlineData(32, 2)]
+    public void ResidualDpcmMatchesScalarOracle(int size, int modeValue)
+    {
+        HevcResidualDpcmMode mode = (HevcResidualDpcmMode)modeValue;
+        int[] actual = new int[size * size];
+        for (int i = 0; i < actual.Length; i++)
+        {
+            actual[i] = (((i * 104729) + (size * 4099)) & 8191) - 4096;
+        }
+
+        int[] expected = (int[])actual.Clone();
+        ApplyResidualDpcmScalar(expected, size, size, mode);
+
+        HevcResidualReconstructor.ApplyResidualDpcm(actual, size, size, mode);
+
+        Assert.True(expected.AsSpan().SequenceEqual(actual));
+    }
+
+    /// <summary>
+    /// Verifies signed residual clipping without clipping the thirty-two-bit recurrence accumulator.
+    /// </summary>
+    /// <param name="modeValue">The numeric differential accumulation direction.</param>
+    [Theory]
+    [InlineData(1)]
+    [InlineData(2)]
+    public void ResidualDpcmClipsStoredSamples(int modeValue)
+    {
+        HevcResidualDpcmMode mode = (HevcResidualDpcmMode)modeValue;
+        int[] actual = new int[32 * 32];
+        actual.AsSpan().Fill(3000);
+        int[] expected = (int[])actual.Clone();
+        ApplyResidualDpcmScalar(expected, 32, 32, mode);
+
+        HevcResidualReconstructor.ApplyResidualDpcm(actual, 32, 32, mode);
+
+        Assert.True(expected.AsSpan().SequenceEqual(actual));
+        Assert.Contains(short.MaxValue, actual);
+    }
+
+    /// <summary>
+    /// Verifies the Range Extensions rotation constraint for non-transformed intra blocks.
+    /// </summary>
+    [Fact]
+    public void RotationRequiresEnabledFourWideIntraBlock()
+    {
+        Assert.True(HevcResidualReconstructor.IsNonTransformedResidualRotated(true, true, 4));
+        Assert.False(HevcResidualReconstructor.IsNonTransformedResidualRotated(false, true, 4));
+        Assert.False(HevcResidualReconstructor.IsNonTransformedResidualRotated(true, false, 4));
+        Assert.False(HevcResidualReconstructor.IsNonTransformedResidualRotated(true, true, 8));
+    }
+
+    /// <summary>
+    /// Verifies implicit residual differential mode selection, including 4:2:2 chroma angle remapping.
+    /// </summary>
+    [Fact]
+    public void ImplicitResidualDpcmFollowsPredictionDirection()
+    {
+        Assert.Equal(HevcResidualDpcmMode.Horizontal, HevcResidualReconstructor.GetImplicitResidualDpcmMode(10, false));
+        Assert.Equal(HevcResidualDpcmMode.Vertical, HevcResidualReconstructor.GetImplicitResidualDpcmMode(26, false));
+        Assert.Equal(HevcResidualDpcmMode.None, HevcResidualReconstructor.GetImplicitResidualDpcmMode(18, false));
+        Assert.Equal(HevcResidualDpcmMode.Horizontal, HevcResidualReconstructor.GetImplicitResidualDpcmMode(10, true));
+        Assert.Equal(HevcResidualDpcmMode.Vertical, HevcResidualReconstructor.GetImplicitResidualDpcmMode(26, true));
+    }
+
+    /// <summary>
+    /// Applies the normative transform-skip normalization as a scalar test oracle.
+    /// </summary>
+    /// <param name="coefficients">The dequantized coefficients.</param>
+    /// <param name="residual">The destination residual block.</param>
+    /// <param name="bitDepth">The reconstructed component precision.</param>
+    /// <param name="maxTransformDynamicRange">The transform dynamic range excluding its sign bit.</param>
+    /// <param name="equivalentLog2TransformSize">The base-two logarithm of the equivalent square transform size.</param>
+    /// <param name="extendedPrecisionProcessingEnabled">Whether extended transform-skip precision applies.</param>
+    /// <param name="rotate">Whether the complete coefficient order is reversed.</param>
+    private static void ApplyTransformSkipScalar(
+        ReadOnlySpan<int> coefficients,
+        Span<int> residual,
+        int bitDepth,
+        int maxTransformDynamicRange,
+        int equivalentLog2TransformSize,
+        bool extendedPrecisionProcessingEnabled,
+        bool rotate)
+    {
+        int shift = maxTransformDynamicRange - bitDepth - equivalentLog2TransformSize;
+        if (extendedPrecisionProcessingEnabled)
+        {
+            shift = Math.Max(0, shift);
+        }
+
+        for (int i = 0; i < coefficients.Length; i++)
+        {
+            int value = coefficients[rotate ? coefficients.Length - 1 - i : i];
+            residual[i] = shift > 0
+                ? (value + (1 << (shift - 1))) >> shift
+                : value << -shift;
+        }
+    }
+
+    /// <summary>
+    /// Applies the normative inverse residual differential recurrence as a scalar test oracle.
+    /// </summary>
+    /// <param name="residual">The residual block in packed raster order.</param>
+    /// <param name="width">The residual-block width.</param>
+    /// <param name="height">The residual-block height.</param>
+    /// <param name="mode">The differential accumulation direction.</param>
+    private static void ApplyResidualDpcmScalar(Span<int> residual, int width, int height, HevcResidualDpcmMode mode)
+    {
+        if (mode == HevcResidualDpcmMode.Vertical)
+        {
+            for (int x = 0; x < width; x++)
+            {
+                int accumulator = residual[x];
+                for (int y = 1; y < height; y++)
+                {
+                    int index = (y * width) + x;
+                    accumulator += residual[index];
+                    residual[index] = Math.Clamp(accumulator, short.MinValue, short.MaxValue);
+                }
+            }
+        }
+        else if (mode == HevcResidualDpcmMode.Horizontal)
+        {
+            for (int y = 0; y < height; y++)
+            {
+                int rowOffset = y * width;
+                int accumulator = residual[rowOffset];
+                for (int x = 1; x < width; x++)
+                {
+                    int index = rowOffset + x;
+                    accumulator += residual[index];
+                    residual[index] = Math.Clamp(accumulator, short.MinValue, short.MaxValue);
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Finds the first unequal element in two equally sized test buffers.
+    /// </summary>
+    /// <param name="expected">The expected values.</param>
+    /// <param name="actual">The actual values.</param>
+    /// <returns>The first unequal index.</returns>
+    private static int FindFirstMismatch(ReadOnlySpan<int> expected, ReadOnlySpan<int> actual)
+    {
+        for (int i = 0; i < expected.Length; i++)
+        {
+            if (expected[i] != actual[i])
+            {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+}