Align HEIF operators with JPEG architecture

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.ByteOutputOperator.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Writes filtered samples to eight-bit plane storage.
+    /// </summary>
+    private readonly struct ByteOutputOperator : IOutputOperator<byte>
+    {
+        /// <inheritdoc/>
+        public static void StoreVector(ref byte destination, int offset, Vector128<short> value, int count)
+        {
+            Vector64<byte> packed = Vector128.Narrow(value.AsUInt16(), Vector128<ushort>.Zero).GetLower();
+            ref byte output = ref Unsafe.Add(ref destination, offset);
+            if (count == 8)
+            {
+                packed.StoreUnsafe(ref output);
+            }
+            else
+            {
+                Unsafe.WriteUnaligned(ref output, packed.AsUInt32().ToScalar());
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void StoreScalar(ref byte destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (byte)value;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.CopyFilterOperator.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Disables both tap groups so the source block is copied unchanged.
+    /// </summary>
+    private readonly struct CopyFilterOperator : IFilterOperator
+    {
+        /// <inheritdoc/>
+        public static bool EnablePrimary => false;
+
+        /// <inheritdoc/>
+        public static bool EnableSecondary => false;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.Operator.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Defines storage-specific writes for one filtered row.
+    /// </summary>
+    /// <typeparam name="TSample">The destination sample storage type.</typeparam>
+    private interface IOutputOperator<TSample>
+        where TSample : unmanaged
+    {
+        /// <summary>
+        /// Stores four or eight filtered samples from the low vector lanes.
+        /// </summary>
+        /// <param name="destination">The first element in the destination plane.</param>
+        /// <param name="offset">The offset of the first sample to write.</param>
+        /// <param name="value">The filtered samples in the low lanes.</param>
+        /// <param name="count">The number of valid lanes.</param>
+        public static abstract void StoreVector(ref TSample destination, int offset, Vector128<short> value, int count);
+
+        /// <summary>
+        /// Stores one filtered sample.
+        /// </summary>
+        /// <param name="destination">The first element in the destination plane.</param>
+        /// <param name="offset">The offset of the sample to write.</param>
+        /// <param name="value">The filtered sample.</param>
+        public static abstract void StoreScalar(ref TSample destination, int offset, int value);
+    }
+
+    /// <summary>
+    /// Defines which groups of directional taps participate in one closed filter kernel.
+    /// </summary>
+    private interface IFilterOperator
+    {
+        /// <summary>
+        /// Gets a value indicating whether the primary directional taps are enabled.
+        /// </summary>
+        public static abstract bool EnablePrimary { get; }
+
+        /// <summary>
+        /// Gets a value indicating whether the secondary off-axis taps are enabled.
+        /// </summary>
+        public static abstract bool EnableSecondary { get; }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryAndSecondaryFilterOperator.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Enables both directional tap groups and their combined clipping rule.
+    /// </summary>
+    private readonly struct PrimaryAndSecondaryFilterOperator : IFilterOperator
+    {
+        /// <inheritdoc/>
+        public static bool EnablePrimary => true;
+
+        /// <inheritdoc/>
+        public static bool EnableSecondary => true;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryFilterOperator.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Enables only the primary directional taps.
+    /// </summary>
+    private readonly struct PrimaryFilterOperator : IFilterOperator
+    {
+        /// <inheritdoc/>
+        public static bool EnablePrimary => true;
+
+        /// <inheritdoc/>
+        public static bool EnableSecondary => false;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.SecondaryFilterOperator.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Enables only the secondary off-axis taps.
+    /// </summary>
+    private readonly struct SecondaryFilterOperator : IFilterOperator
+    {
+        /// <inheritdoc/>
+        public static bool EnablePrimary => false;
+
+        /// <inheritdoc/>
+        public static bool EnableSecondary => true;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.UInt16OutputOperator.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
+
+internal static partial class Av1CdefFilter
+{
+    /// <summary>
+    /// Writes filtered samples to 16-bit plane storage.
+    /// </summary>
+    private readonly struct UInt16OutputOperator : IOutputOperator<ushort>
+    {
+        /// <inheritdoc/>
+        public static void StoreVector(ref ushort destination, int offset, Vector128<short> value, int count)
+        {
+            ref ushort output = ref Unsafe.Add(ref destination, offset);
+            if (count == 8)
+            {
+                value.AsUInt16().StoreUnsafe(ref output);
+            }
+            else
+            {
+                ref byte outputBytes = ref Unsafe.As<ushort, byte>(ref output);
+                Unsafe.WriteUnaligned(ref outputBytes, value.AsUInt64().GetLower().ToScalar());
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void StoreScalar(ref ushort destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (ushort)value;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs

@@ -18,54 +18,13 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
 /// one eight-by-eight block per 128-bit lane so AVX2 can evaluate two independent blocks together. Scalar kernels retain
 /// the same constrain, clipping, and tie-breaking rules for unsupported hardware and partial edge blocks.
 /// </remarks>
-internal static class Av1CdefFilter
+internal static partial class Av1CdefFilter
 {
     /// <summary>
     /// The sample value used in the bordered source plane for neighbors outside the coded frame.
     /// </summary>
     public const ushort VeryLarge = 0x4000;
 
-    /// <summary>
-    /// Defines storage-specific writes for one filtered row.
-    /// </summary>
-    /// <typeparam name="TSample">The destination sample storage type.</typeparam>
-    private interface IOutputOperator<TSample>
-        where TSample : unmanaged
-    {
-        /// <summary>
-        /// Stores four or eight filtered samples from the low vector lanes.
-        /// </summary>
-        /// <param name="destination">The first element in the destination plane.</param>
-        /// <param name="offset">The offset of the first sample to write.</param>
-        /// <param name="value">The filtered samples in the low lanes.</param>
-        /// <param name="count">The number of valid lanes.</param>
-        public static abstract void StoreVector(ref TSample destination, int offset, Vector128<short> value, int count);
-
-        /// <summary>
-        /// Stores one filtered sample.
-        /// </summary>
-        /// <param name="destination">The first element in the destination plane.</param>
-        /// <param name="offset">The offset of the sample to write.</param>
-        /// <param name="value">The filtered sample.</param>
-        public static abstract void StoreScalar(ref TSample destination, int offset, int value);
-    }
-
-    /// <summary>
-    /// Defines which groups of directional taps participate in one closed filter kernel.
-    /// </summary>
-    private interface IFilterOperator
-    {
-        /// <summary>
-        /// Gets a value indicating whether the primary directional taps are enabled.
-        /// </summary>
-        public static abstract bool EnablePrimary { get; }
-
-        /// <summary>
-        /// Gets a value indicating whether the secondary off-axis taps are enabled.
-        /// </summary>
-        public static abstract bool EnableSecondary { get; }
-    }
-
     /// <summary>
     /// Copies an eight-bit sample rectangle into the 16-bit CDEF working plane.
     /// </summary>
@@ -1732,100 +1691,4 @@ internal static class Av1CdefFilter
         6 => tap == 0 ? stride : 2 * stride,
         _ => tap == 0 ? stride : (2 * stride) - 1
     };
-
-    /// <summary>
-    /// Enables both directional tap groups and their combined clipping rule.
-    /// </summary>
-    private readonly struct PrimaryAndSecondaryFilterOperator : IFilterOperator
-    {
-        /// <inheritdoc/>
-        public static bool EnablePrimary => true;
-
-        /// <inheritdoc/>
-        public static bool EnableSecondary => true;
-    }
-
-    /// <summary>
-    /// Enables only the primary directional taps.
-    /// </summary>
-    private readonly struct PrimaryFilterOperator : IFilterOperator
-    {
-        /// <inheritdoc/>
-        public static bool EnablePrimary => true;
-
-        /// <inheritdoc/>
-        public static bool EnableSecondary => false;
-    }
-
-    /// <summary>
-    /// Enables only the secondary off-axis taps.
-    /// </summary>
-    private readonly struct SecondaryFilterOperator : IFilterOperator
-    {
-        /// <inheritdoc/>
-        public static bool EnablePrimary => false;
-
-        /// <inheritdoc/>
-        public static bool EnableSecondary => true;
-    }
-
-    /// <summary>
-    /// Disables both tap groups so the source block is copied unchanged.
-    /// </summary>
-    private readonly struct CopyFilterOperator : IFilterOperator
-    {
-        /// <inheritdoc/>
-        public static bool EnablePrimary => false;
-
-        /// <inheritdoc/>
-        public static bool EnableSecondary => false;
-    }
-
-    /// <summary>
-    /// Writes filtered samples to eight-bit plane storage.
-    /// </summary>
-    private readonly struct ByteOutputOperator : IOutputOperator<byte>
-    {
-        /// <inheritdoc/>
-        public static void StoreVector(ref byte destination, int offset, Vector128<short> value, int count)
-        {
-            Vector64<byte> packed = Vector128.Narrow(value.AsUInt16(), Vector128<ushort>.Zero).GetLower();
-            ref byte output = ref Unsafe.Add(ref destination, offset);
-            if (count == 8)
-            {
-                packed.StoreUnsafe(ref output);
-            }
-            else
-            {
-                Unsafe.WriteUnaligned(ref output, packed.AsUInt32().ToScalar());
-            }
-        }
-
-        /// <inheritdoc/>
-        public static void StoreScalar(ref byte destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (byte)value;
-    }
-
-    /// <summary>
-    /// Writes filtered samples to 16-bit plane storage.
-    /// </summary>
-    private readonly struct UInt16OutputOperator : IOutputOperator<ushort>
-    {
-        /// <inheritdoc/>
-        public static void StoreVector(ref ushort destination, int offset, Vector128<short> value, int count)
-        {
-            ref ushort output = ref Unsafe.Add(ref destination, offset);
-            if (count == 8)
-            {
-                value.AsUInt16().StoreUnsafe(ref output);
-            }
-            else
-            {
-                ref byte outputBytes = ref Unsafe.As<ushort, byte>(ref output);
-                Unsafe.WriteUnaligned(ref outputBytes, value.AsUInt64().GetLower().ToScalar());
-            }
-        }
-
-        /// <inheritdoc/>
-        public static void StoreScalar(ref ushort destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (ushort)value;
-    }
 }

src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs

@@ -271,10 +271,10 @@ internal static class Av1FilmGrainNoise
             for (; column <= vectorEnd; column += Vector256<int>.Count)
             {
                 ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column);
-                Vector256<int> source = Av1FilmGrainSampleOperator<TSample>.Load8(ref destination);
+                Vector256<int> source = Av1FilmGrainSampleOperations<TSample>.Load8(ref destination);
                 Vector256<int> grainValues = Vector256.LoadUnsafe(ref grainBase, (nuint)(grainRowOffset + column));
                 Vector256<int> result = AddNoise(source, grainValues, scaling, bitDepth, roundingOffset, scalingShift, minimum, maximum);
-                Av1FilmGrainSampleOperator<TSample>.Store8(ref destination, result);
+                Av1FilmGrainSampleOperations<TSample>.Store8(ref destination, result);
             }
 
             ApplyLumaScalar(
@@ -326,10 +326,10 @@ internal static class Av1FilmGrainNoise
             for (; column <= vectorEnd; column += Vector128<int>.Count)
             {
                 ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column);
-                Vector128<int> source = Av1FilmGrainSampleOperator<TSample>.Load4(ref destination);
+                Vector128<int> source = Av1FilmGrainSampleOperations<TSample>.Load4(ref destination);
                 Vector128<int> grainValues = Vector128.LoadUnsafe(ref grainBase, (nuint)(grainRowOffset + column));
                 Vector128<int> result = AddNoise(source, grainValues, scaling, bitDepth, roundingOffset, scalingShift, minimum, maximum);
-                Av1FilmGrainSampleOperator<TSample>.Store4(ref destination, result);
+                Av1FilmGrainSampleOperations<TSample>.Store4(ref destination, result);
             }
 
             ApplyLumaScalar(
@@ -375,10 +375,10 @@ internal static class Av1FilmGrainNoise
             for (int column = 0; column < width; column++)
             {
                 ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column);
-                int source = Av1FilmGrainSampleOperator<TSample>.Load(ref destination);
+                int source = Av1FilmGrainSampleOperations<TSample>.Load(ref destination);
                 int scale = ScaleLookup(scaling, source, bitDepth);
                 int value = source + (((scale * Unsafe.Add(ref grainBase, grainRowOffset + column)) + roundingOffset) >> scalingShift);
-                Av1FilmGrainSampleOperator<TSample>.Store(ref destination, Av1Math.Clamp(value, minimum, maximum));
+                Av1FilmGrainSampleOperations<TSample>.Store(ref destination, Av1Math.Clamp(value, minimum, maximum));
             }
         }
     }
@@ -577,11 +577,11 @@ internal static class Av1FilmGrainNoise
             for (; column <= vectorEnd; column += Vector256<int>.Count)
             {
                 ref TSample lumaSource = ref Unsafe.Add(ref lumaRow, column << subsamplingX);
-                Vector256<int> averageLuma = Av1FilmGrainSampleOperator<TSample>.LoadChromaLuma8(ref lumaSource, subsamplingX);
+                Vector256<int> averageLuma = Av1FilmGrainSampleOperations<TSample>.LoadChromaLuma8(ref lumaSource, subsamplingX);
                 if (applyCb)
                 {
                     ref TSample destination = ref Unsafe.Add(ref cbBase, chromaRowOffset + column);
-                    Vector256<int> source = Av1FilmGrainSampleOperator<TSample>.Load8(ref destination);
+                    Vector256<int> source = Av1FilmGrainSampleOperations<TSample>.Load8(ref destination);
                     Vector256<int> scalingIndex = ((averageLuma * cbLumaMultiplier) + (source * cbMultiplier)) >> 6;
                     scalingIndex = Vector256.Min(Vector256.Max(scalingIndex + Vector256.Create(cbOffset), zero), maximumIndex);
                     Vector256<int> grainValues = Vector256.LoadUnsafe(ref cbGrainBase, (nuint)(grainRowOffset + column));
@@ -596,13 +596,13 @@ internal static class Av1FilmGrainNoise
                         minimum,
                         maximum);
 
-                    Av1FilmGrainSampleOperator<TSample>.Store8(ref destination, result);
+                    Av1FilmGrainSampleOperations<TSample>.Store8(ref destination, result);
                 }
 
                 if (applyCr)
                 {
                     ref TSample destination = ref Unsafe.Add(ref crBase, chromaRowOffset + column);
-                    Vector256<int> source = Av1FilmGrainSampleOperator<TSample>.Load8(ref destination);
+                    Vector256<int> source = Av1FilmGrainSampleOperations<TSample>.Load8(ref destination);
                     Vector256<int> scalingIndex = ((averageLuma * crLumaMultiplier) + (source * crMultiplier)) >> 6;
                     scalingIndex = Vector256.Min(Vector256.Max(scalingIndex + Vector256.Create(crOffset), zero), maximumIndex);
                     Vector256<int> grainValues = Vector256.LoadUnsafe(ref crGrainBase, (nuint)(grainRowOffset + column));
@@ -617,7 +617,7 @@ internal static class Av1FilmGrainNoise
                         minimum,
                         maximum);
 
-                    Av1FilmGrainSampleOperator<TSample>.Store8(ref destination, result);
+                    Av1FilmGrainSampleOperations<TSample>.Store8(ref destination, result);
                 }
             }
 
@@ -708,11 +708,11 @@ internal static class Av1FilmGrainNoise
             for (; column <= vectorEnd; column += Vector128<int>.Count)
             {
                 ref TSample lumaSource = ref Unsafe.Add(ref lumaRow, column << subsamplingX);
-                Vector128<int> averageLuma = Av1FilmGrainSampleOperator<TSample>.LoadChromaLuma4(ref lumaSource, subsamplingX);
+                Vector128<int> averageLuma = Av1FilmGrainSampleOperations<TSample>.LoadChromaLuma4(ref lumaSource, subsamplingX);
                 if (applyCb)
                 {
                     ref TSample destination = ref Unsafe.Add(ref cbBase, chromaRowOffset + column);
-                    Vector128<int> source = Av1FilmGrainSampleOperator<TSample>.Load4(ref destination);
+                    Vector128<int> source = Av1FilmGrainSampleOperations<TSample>.Load4(ref destination);
                     Vector128<int> scalingIndex = ((averageLuma * cbLumaMultiplier) + (source * cbMultiplier)) >> 6;
                     scalingIndex = Vector128.Min(Vector128.Max(scalingIndex + Vector128.Create(cbOffset), zero), maximumIndex);
                     Vector128<int> grainValues = Vector128.LoadUnsafe(ref cbGrainBase, (nuint)(grainRowOffset + column));
@@ -727,13 +727,13 @@ internal static class Av1FilmGrainNoise
                         minimum,
                         maximum);
 
-                    Av1FilmGrainSampleOperator<TSample>.Store4(ref destination, result);
+                    Av1FilmGrainSampleOperations<TSample>.Store4(ref destination, result);
                 }
 
                 if (applyCr)
                 {
                     ref TSample destination = ref Unsafe.Add(ref crBase, chromaRowOffset + column);
-                    Vector128<int> source = Av1FilmGrainSampleOperator<TSample>.Load4(ref destination);
+                    Vector128<int> source = Av1FilmGrainSampleOperations<TSample>.Load4(ref destination);
                     Vector128<int> scalingIndex = ((averageLuma * crLumaMultiplier) + (source * crMultiplier)) >> 6;
                     scalingIndex = Vector128.Min(Vector128.Max(scalingIndex + Vector128.Create(crOffset), zero), maximumIndex);
                     Vector128<int> grainValues = Vector128.LoadUnsafe(ref crGrainBase, (nuint)(grainRowOffset + column));
@@ -748,7 +748,7 @@ internal static class Av1FilmGrainNoise
                         minimum,
                         maximum);
 
-                    Av1FilmGrainSampleOperator<TSample>.Store4(ref destination, result);
+                    Av1FilmGrainSampleOperations<TSample>.Store4(ref destination, result);
                 }
             }
 
@@ -831,10 +831,10 @@ internal static class Av1FilmGrainNoise
             for (int column = 0; column < width; column++)
             {
                 int lumaOffset = lumaRowOffset + (column << subsamplingX);
-                int averageLuma = Av1FilmGrainSampleOperator<TSample>.Load(ref Unsafe.Add(ref lumaBase, lumaOffset));
+                int averageLuma = Av1FilmGrainSampleOperations<TSample>.Load(ref Unsafe.Add(ref lumaBase, lumaOffset));
                 if (subsamplingX != 0)
                 {
-                    averageLuma = (averageLuma + Av1FilmGrainSampleOperator<TSample>.Load(ref Unsafe.Add(ref lumaBase, lumaOffset + 1)) + 1) >> 1;
+                    averageLuma = (averageLuma + Av1FilmGrainSampleOperations<TSample>.Load(ref Unsafe.Add(ref lumaBase, lumaOffset + 1)) + 1) >> 1;
                 }
 
                 int chromaOffset = chromaRowOffset + column;
@@ -842,7 +842,7 @@ internal static class Av1FilmGrainNoise
                 if (applyCb)
                 {
                     ref TSample destination = ref Unsafe.Add(ref cbBase, chromaOffset);
-                    int source = Av1FilmGrainSampleOperator<TSample>.Load(ref destination);
+                    int source = Av1FilmGrainSampleOperations<TSample>.Load(ref destination);
                     int scalingIndex = Av1Math.Clamp(
                         (((averageLuma * cbLumaMultiplier) + (source * cbMultiplier)) >> 6) + cbOffset,
                         0,
@@ -850,13 +850,13 @@ internal static class Av1FilmGrainNoise
 
                     int scale = ScaleLookup(scalingCb, scalingIndex, bitDepth);
                     int value = source + (((scale * Unsafe.Add(ref cbGrainBase, grainOffset)) + roundingOffset) >> scalingShift);
-                    Av1FilmGrainSampleOperator<TSample>.Store(ref destination, Av1Math.Clamp(value, minimum, maximum));
+                    Av1FilmGrainSampleOperations<TSample>.Store(ref destination, Av1Math.Clamp(value, minimum, maximum));
                 }
 
                 if (applyCr)
                 {
                     ref TSample destination = ref Unsafe.Add(ref crBase, chromaOffset);
-                    int source = Av1FilmGrainSampleOperator<TSample>.Load(ref destination);
+                    int source = Av1FilmGrainSampleOperations<TSample>.Load(ref destination);
                     int scalingIndex = Av1Math.Clamp(
                         (((averageLuma * crLumaMultiplier) + (source * crMultiplier)) >> 6) + crOffset,
                         0,
@@ -864,7 +864,7 @@ internal static class Av1FilmGrainNoise
 
                     int scale = ScaleLookup(scalingCr, scalingIndex, bitDepth);
                     int value = source + (((scale * Unsafe.Add(ref crGrainBase, grainOffset)) + roundingOffset) >> scalingShift);
-                    Av1FilmGrainSampleOperator<TSample>.Store(ref destination, Av1Math.Clamp(value, minimum, maximum));
+                    Av1FilmGrainSampleOperations<TSample>.Store(ref destination, Av1Math.Clamp(value, minimum, maximum));
                 }
             }
         }

src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperator.cs → src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperations.cs

@@ -18,7 +18,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain;
 /// row loops. All arithmetic uses signed 32-bit lanes; decoded samples are nonnegative and at most twelve bits, making
 /// the intermediate signed 16-bit views safe wherever pairwise operations require them.
 /// </remarks>
-internal readonly struct Av1FilmGrainSampleOperator<TSample>
+internal readonly struct Av1FilmGrainSampleOperations<TSample>
     where TSample : unmanaged
 {
     /// <summary>

src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalByteEdgeOperator.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
+
+internal static partial class Av1DeblockingFilter
+{
+    /// <summary>
+    /// Accesses four columns across a horizontal edge in eight-bit storage.
+    /// </summary>
+    private readonly struct HorizontalByteEdgeOperator : IEdgeOperator<byte>
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> LoadVector(ref byte samples, int q0Offset, int stride, int distance)
+        {
+            ref byte source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride));
+            uint packed = Unsafe.ReadUnaligned<uint>(ref source);
+            Vector128<ushort> widened = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
+            return Vector128.WidenLower(widened).AsInt32();
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128<int> value)
+        {
+            Vector128<ushort> narrowed16 = Vector128.Narrow(value.AsUInt32(), Vector128<uint>.Zero);
+            Vector128<byte> narrowed8 = Vector128.Narrow(narrowed16, Vector128<ushort>.Zero);
+            Unsafe.WriteUnaligned(ref Unsafe.Add(ref samples, q0Offset + (distance * stride)), narrowed8.AsUInt32().ToScalar());
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index)
+            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index);
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value)
+            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (byte)value;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalUInt16EdgeOperator.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
+
+internal static partial class Av1DeblockingFilter
+{
+    /// <summary>
+    /// Accesses four columns across a horizontal edge in 16-bit storage.
+    /// </summary>
+    private readonly struct HorizontalUInt16EdgeOperator : IEdgeOperator<ushort>
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> LoadVector(ref ushort samples, int q0Offset, int stride, int distance)
+        {
+            ref ushort source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride));
+            ulong packed = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<ushort, byte>(ref source));
+            return Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsUInt16()).AsInt32();
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128<int> value)
+        {
+            Vector64<ushort> narrowed = Vector128.Narrow(value, Vector128<int>.Zero).AsUInt16().GetLower();
+            ref byte destination = ref Unsafe.As<ushort, byte>(ref Unsafe.Add(ref samples, q0Offset + (distance * stride)));
+            Unsafe.WriteUnaligned(ref destination, narrowed.AsUInt64().ToScalar());
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index)
+            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index);
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value)
+            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (ushort)value;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.Operator.cs

@@ -0,0 +1,59 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
+
+internal static partial class Av1DeblockingFilter
+{
+    /// <summary>
+    /// Defines orientation- and storage-specific access to the four samples running along one edge segment.
+    /// </summary>
+    /// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
+    private interface IEdgeOperator<TSample>
+        where TSample : unmanaged
+    {
+        /// <summary>
+        /// Loads four samples at one signed distance across the edge.
+        /// </summary>
+        /// <param name="samples">The first element in the plane storage.</param>
+        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
+        /// <param name="stride">The number of samples between adjacent rows.</param>
+        /// <param name="distance">The signed sample distance from Q0.</param>
+        /// <returns>The widened samples ordered along the edge.</returns>
+        public static abstract Vector128<int> LoadVector(ref TSample samples, int q0Offset, int stride, int distance);
+
+        /// <summary>
+        /// Stores four samples at one signed distance across the edge.
+        /// </summary>
+        /// <param name="samples">The first element in the plane storage.</param>
+        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
+        /// <param name="stride">The number of samples between adjacent rows.</param>
+        /// <param name="distance">The signed sample distance from Q0.</param>
+        /// <param name="value">The widened samples ordered along the edge.</param>
+        public static abstract void StoreVector(ref TSample samples, int q0Offset, int stride, int distance, Vector128<int> value);
+
+        /// <summary>
+        /// Loads one sample at a signed distance across and an offset along the edge.
+        /// </summary>
+        /// <param name="samples">The first element in the plane storage.</param>
+        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
+        /// <param name="stride">The number of samples between adjacent rows.</param>
+        /// <param name="distance">The signed sample distance from Q0.</param>
+        /// <param name="index">The sample offset along the edge.</param>
+        /// <returns>The selected sample.</returns>
+        public static abstract int LoadScalar(ref TSample samples, int q0Offset, int stride, int distance, int index);
+
+        /// <summary>
+        /// Stores one sample at a signed distance across and an offset along the edge.
+        /// </summary>
+        /// <param name="samples">The first element in the plane storage.</param>
+        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
+        /// <param name="stride">The number of samples between adjacent rows.</param>
+        /// <param name="distance">The signed sample distance from Q0.</param>
+        /// <param name="index">The sample offset along the edge.</param>
+        /// <param name="value">The filtered sample.</param>
+        public static abstract void StoreScalar(ref TSample samples, int q0Offset, int stride, int distance, int index, int value);
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalByteEdgeOperator.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
+
+internal static partial class Av1DeblockingFilter
+{
+    /// <summary>
+    /// Accesses four rows across a vertical edge in eight-bit storage.
+    /// </summary>
+    private readonly struct VerticalByteEdgeOperator : IEdgeOperator<byte>
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> LoadVector(ref byte samples, int q0Offset, int stride, int distance)
+            => Vector128.Create(
+                (int)Unsafe.Add(ref samples, q0Offset + distance),
+                Unsafe.Add(ref samples, q0Offset + stride + distance),
+                Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance),
+                Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance));
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128<int> value)
+        {
+            Unsafe.Add(ref samples, q0Offset + distance) = (byte)value.GetElement(0);
+            Unsafe.Add(ref samples, q0Offset + stride + distance) = (byte)value.GetElement(1);
+            Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (byte)value.GetElement(2);
+            Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (byte)value.GetElement(3);
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index)
+            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance);
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value)
+            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (byte)value;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalUInt16EdgeOperator.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
+
+internal static partial class Av1DeblockingFilter
+{
+    /// <summary>
+    /// Accesses four rows across a vertical edge in 16-bit storage.
+    /// </summary>
+    private readonly struct VerticalUInt16EdgeOperator : IEdgeOperator<ushort>
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> LoadVector(ref ushort samples, int q0Offset, int stride, int distance)
+            => Vector128.Create(
+                (int)Unsafe.Add(ref samples, q0Offset + distance),
+                Unsafe.Add(ref samples, q0Offset + stride + distance),
+                Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance),
+                Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance));
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128<int> value)
+        {
+            Unsafe.Add(ref samples, q0Offset + distance) = (ushort)value.GetElement(0);
+            Unsafe.Add(ref samples, q0Offset + stride + distance) = (ushort)value.GetElement(1);
+            Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (ushort)value.GetElement(2);
+            Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (ushort)value.GetElement(3);
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index)
+            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance);
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value)
+            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (ushort)value;
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs

@@ -16,58 +16,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
 /// lane-wise. Conditional selection preserves unfiltered lanes while allowing four adjacent edge positions to share
 /// one kernel invocation.
 /// </remarks>
-internal static class Av1DeblockingFilter
+internal static partial class Av1DeblockingFilter
 {
-    /// <summary>
-    /// Defines orientation- and storage-specific access to the four samples running along one edge segment.
-    /// </summary>
-    /// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
-    private interface IEdgeOperator<TSample>
-        where TSample : unmanaged
-    {
-        /// <summary>
-        /// Loads four samples at one signed distance across the edge.
-        /// </summary>
-        /// <param name="samples">The first element in the plane storage.</param>
-        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
-        /// <param name="stride">The number of samples between adjacent rows.</param>
-        /// <param name="distance">The signed sample distance from Q0.</param>
-        /// <returns>The widened samples ordered along the edge.</returns>
-        public static abstract Vector128<int> LoadVector(ref TSample samples, int q0Offset, int stride, int distance);
-
-        /// <summary>
-        /// Stores four samples at one signed distance across the edge.
-        /// </summary>
-        /// <param name="samples">The first element in the plane storage.</param>
-        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
-        /// <param name="stride">The number of samples between adjacent rows.</param>
-        /// <param name="distance">The signed sample distance from Q0.</param>
-        /// <param name="value">The widened samples ordered along the edge.</param>
-        public static abstract void StoreVector(ref TSample samples, int q0Offset, int stride, int distance, Vector128<int> value);
-
-        /// <summary>
-        /// Loads one sample at a signed distance across and an offset along the edge.
-        /// </summary>
-        /// <param name="samples">The first element in the plane storage.</param>
-        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
-        /// <param name="stride">The number of samples between adjacent rows.</param>
-        /// <param name="distance">The signed sample distance from Q0.</param>
-        /// <param name="index">The sample offset along the edge.</param>
-        /// <returns>The selected sample.</returns>
-        public static abstract int LoadScalar(ref TSample samples, int q0Offset, int stride, int distance, int index);
-
-        /// <summary>
-        /// Stores one sample at a signed distance across and an offset along the edge.
-        /// </summary>
-        /// <param name="samples">The first element in the plane storage.</param>
-        /// <param name="q0Offset">The offset of the first Q-side sample.</param>
-        /// <param name="stride">The number of samples between adjacent rows.</param>
-        /// <param name="distance">The signed sample distance from Q0.</param>
-        /// <param name="index">The sample offset along the edge.</param>
-        /// <param name="value">The filtered sample.</param>
-        public static abstract void StoreScalar(ref TSample samples, int q0Offset, int stride, int distance, int index, int value);
-    }
-
     /// <summary>
     /// Filters four rows crossing one vertical boundary in eight-bit storage.
     /// </summary>
@@ -943,143 +893,4 @@ internal static class Av1DeblockingFilter
     /// <returns>The rounded quotient.</returns>
     private static int RoundPowerOfTwo(int value, int bitCount)
         => (value + (1 << (bitCount - 1))) >> bitCount;
-
-    /// <summary>
-    /// Accesses four rows across a vertical edge in eight-bit storage.
-    /// </summary>
-    private readonly struct VerticalByteEdgeOperator : IEdgeOperator<byte>
-    {
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static Vector128<int> LoadVector(ref byte samples, int q0Offset, int stride, int distance)
-            => Vector128.Create(
-                (int)Unsafe.Add(ref samples, q0Offset + distance),
-                Unsafe.Add(ref samples, q0Offset + stride + distance),
-                Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance),
-                Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance));
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128<int> value)
-        {
-            Unsafe.Add(ref samples, q0Offset + distance) = (byte)value.GetElement(0);
-            Unsafe.Add(ref samples, q0Offset + stride + distance) = (byte)value.GetElement(1);
-            Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (byte)value.GetElement(2);
-            Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (byte)value.GetElement(3);
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index)
-            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance);
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value)
-            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (byte)value;
-    }
-
-    /// <summary>
-    /// Accesses four columns across a horizontal edge in eight-bit storage.
-    /// </summary>
-    private readonly struct HorizontalByteEdgeOperator : IEdgeOperator<byte>
-    {
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static Vector128<int> LoadVector(ref byte samples, int q0Offset, int stride, int distance)
-        {
-            ref byte source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride));
-            uint packed = Unsafe.ReadUnaligned<uint>(ref source);
-            Vector128<ushort> widened = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
-            return Vector128.WidenLower(widened).AsInt32();
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128<int> value)
-        {
-            Vector128<ushort> narrowed16 = Vector128.Narrow(value.AsUInt32(), Vector128<uint>.Zero);
-            Vector128<byte> narrowed8 = Vector128.Narrow(narrowed16, Vector128<ushort>.Zero);
-            Unsafe.WriteUnaligned(ref Unsafe.Add(ref samples, q0Offset + (distance * stride)), narrowed8.AsUInt32().ToScalar());
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index)
-            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index);
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value)
-            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (byte)value;
-    }
-
-    /// <summary>
-    /// Accesses four rows across a vertical edge in 16-bit storage.
-    /// </summary>
-    private readonly struct VerticalUInt16EdgeOperator : IEdgeOperator<ushort>
-    {
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static Vector128<int> LoadVector(ref ushort samples, int q0Offset, int stride, int distance)
-            => Vector128.Create(
-                (int)Unsafe.Add(ref samples, q0Offset + distance),
-                Unsafe.Add(ref samples, q0Offset + stride + distance),
-                Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance),
-                Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance));
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128<int> value)
-        {
-            Unsafe.Add(ref samples, q0Offset + distance) = (ushort)value.GetElement(0);
-            Unsafe.Add(ref samples, q0Offset + stride + distance) = (ushort)value.GetElement(1);
-            Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (ushort)value.GetElement(2);
-            Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (ushort)value.GetElement(3);
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index)
-            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance);
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value)
-            => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (ushort)value;
-    }
-
-    /// <summary>
-    /// Accesses four columns across a horizontal edge in 16-bit storage.
-    /// </summary>
-    private readonly struct HorizontalUInt16EdgeOperator : IEdgeOperator<ushort>
-    {
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static Vector128<int> LoadVector(ref ushort samples, int q0Offset, int stride, int distance)
-        {
-            ref ushort source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride));
-            ulong packed = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<ushort, byte>(ref source));
-            return Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsUInt16()).AsInt32();
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128<int> value)
-        {
-            Vector64<ushort> narrowed = Vector128.Narrow(value, Vector128<int>.Zero).AsUInt16().GetLower();
-            ref byte destination = ref Unsafe.As<ushort, byte>(ref Unsafe.Add(ref samples, q0Offset + (distance * stride)));
-            Unsafe.WriteUnaligned(ref destination, narrowed.AsUInt64().ToScalar());
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index)
-            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index);
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value)
-            => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (ushort)value;
-    }
 }

src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs

@@ -18,7 +18,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
 /// Eight-bit blocks use saturating 16-bit stages where their normative ranges permit it; high-bit-depth and scalar
 /// fallback paths retain 32-bit stages. Both representations produce the same row-major coefficient contract.
 /// </remarks>
-internal static class Av1ForwardTransformer
+internal static partial class Av1ForwardTransformer
 {
     /// <summary>
     /// Resolves and applies the configured two-dimensional AV1 forward transform.
@@ -65,40 +65,40 @@ internal static class Av1ForwardTransformer
         switch (config.TransformFunctionTypeColumn)
         {
             case Av1TransformFunctionType.Dct4:
-                DispatchRow<Av1Dct4Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Dct4Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct8:
-                DispatchRow<Av1Dct8Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Dct8Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct16:
-                DispatchRow<Av1Dct16Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Dct16Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct32:
-                DispatchRow<Av1Dct32Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Dct32Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct64:
-                DispatchRow<Av1Dct64Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Dct64Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Adst4:
-                DispatchRow<Av1Adst4Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Adst4Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Adst8:
-                DispatchRow<Av1Adst8Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Adst8Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Adst16:
-                DispatchRow<Av1Adst16Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Adst16Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity4:
-                DispatchRow<Av1Identity4Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Identity4Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity8:
-                DispatchRow<Av1Identity8Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Identity8Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity16:
-                DispatchRow<Av1Identity16Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Identity16Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity32:
-                DispatchRow<Av1Identity32Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                DispatchRow<Identity32Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             default:
                 throw new InvalidImageContentException($"The {config.TransformFunctionTypeColumn} column transform is not valid for {config.TransformSize}.");
@@ -127,40 +127,40 @@ internal static class Av1ForwardTransformer
         switch (config.TransformFunctionTypeRow)
         {
             case Av1TransformFunctionType.Dct4:
-                Transform2d<TColumnOperator, Av1Dct4Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Dct4Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct8:
-                Transform2d<TColumnOperator, Av1Dct8Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Dct8Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct16:
-                Transform2d<TColumnOperator, Av1Dct16Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Dct16Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct32:
-                Transform2d<TColumnOperator, Av1Dct32Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Dct32Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Dct64:
-                Transform2d<TColumnOperator, Av1Dct64Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Dct64Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Adst4:
-                Transform2d<TColumnOperator, Av1Adst4Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Adst4Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Adst8:
-                Transform2d<TColumnOperator, Av1Adst8Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Adst8Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Adst16:
-                Transform2d<TColumnOperator, Av1Adst16Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Adst16Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity4:
-                Transform2d<TColumnOperator, Av1Identity4Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Identity4Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity8:
-                Transform2d<TColumnOperator, Av1Identity8Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Identity8Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity16:
-                Transform2d<TColumnOperator, Av1Identity16Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Identity16Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             case Av1TransformFunctionType.Identity32:
-                Transform2d<TColumnOperator, Av1Identity32Forward1dOperator>(input, coefficients, stride, bitDepth, ref config, workspace);
+                Transform2d<TColumnOperator, Identity32Operator>(input, coefficients, stride, bitDepth, ref config, workspace);
                 break;
             default:
                 throw new InvalidImageContentException($"The {config.TransformFunctionTypeRow} row transform is not valid for {config.TransformSize}.");
@@ -545,7 +545,7 @@ internal static class Av1ForwardTransformer
     /// <summary>
     /// Applies one packed transform axis using the widest efficient lane count available for the block.
     /// </summary>
-    /// <typeparam name="TOperator">The transform operator applied to each independent axis.</typeparam>
+    /// <typeparam name="TOperator">The semantic transform operator.</typeparam>
     /// <param name="buffer">The packed transform block.</param>
     /// <param name="transformCount">The number of independent axes.</param>
     /// <param name="inputStride">The number of packed values between input positions.</param>
@@ -563,23 +563,23 @@ internal static class Av1ForwardTransformer
     {
         if (Avx512BW.IsSupported && transformCount >= Vector512<short>.Count)
         {
-            TransformAxis<TOperator, short, Vector512<short>>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+            TransformPackedVector512<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
             return;
         }
 
         if (Avx2.IsSupported && transformCount >= Vector256<short>.Count)
         {
-            TransformAxis<TOperator, short, Vector256<short>>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+            TransformPackedVector256<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
             return;
         }
 
-        TransformAxis<TOperator, short, Vector128<short>>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+        TransformPackedVector128<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
     }
 
     /// <summary>
-    /// Applies one signed thirty-two-bit transform axis using the widest efficient lane count available for the block.
+    /// Applies one expanded transform axis using the widest efficient lane count available for the block.
     /// </summary>
-    /// <typeparam name="TOperator">The transform operator applied to each independent axis.</typeparam>
+    /// <typeparam name="TOperator">The semantic transform operator.</typeparam>
     /// <param name="buffer">The expanded transform block.</param>
     /// <param name="transformCount">The number of independent axes.</param>
     /// <param name="inputStride">The number of expanded values between input positions.</param>
@@ -597,66 +597,218 @@ internal static class Av1ForwardTransformer
     {
         if (Vector512.IsHardwareAccelerated && transformCount >= Vector512<int>.Count)
         {
-            TransformAxis<TOperator, int, Vector512<int>>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+            TransformExpandedVector512<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
             return;
         }
 
         if (Vector256.IsHardwareAccelerated && transformCount >= Vector256<int>.Count)
         {
-            TransformAxis<TOperator, int, Vector256<int>>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+            TransformExpandedVector256<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
             return;
         }
 
         if (Vector128.IsHardwareAccelerated && transformCount >= Vector128<int>.Count)
         {
-            TransformAxis<TOperator, int, Vector128<int>>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+            TransformExpandedVector128<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
             return;
         }
 
-        TransformAxis<TOperator, int, int>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
+        TransformExpandedScalar<TOperator>(buffer, transformCount, inputStride, outputStride, cosBit, workspace);
     }
 
     /// <summary>
-    /// Applies one transform stage network to independent axes held in scalar or SIMD lanes.
+    /// Applies a packed transform to thirty-two independent axes.
     /// </summary>
-    /// <typeparam name="TOperator">The transform operator applied to each independent axis.</typeparam>
-    /// <typeparam name="TElement">The scalar storage element.</typeparam>
-    /// <typeparam name="TValue">The scalar or SIMD value containing independent transform axes.</typeparam>
-    /// <param name="buffer">The transform block.</param>
-    /// <param name="transformCount">The number of independent axes.</param>
-    /// <param name="inputStride">The number of storage elements between input positions.</param>
-    /// <param name="outputStride">The number of storage elements between output positions.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="workspace">The reusable transform-stage workspace.</param>
-    private static void TransformAxis<TOperator, TElement, TValue>(
-        Span<TElement> buffer,
+    private static void TransformPackedVector512<TOperator>(
+        Span<short> buffer,
+        int transformCount,
+        int inputStride,
+        int outputStride,
+        int cosBit,
+        Span<int> workspace)
+        where TOperator : struct, IAv1ForwardTransform1dOperator
+    {
+        ref Av1TransformVector<Vector512<short>> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<Vector512<short>>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<Vector512<short>> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref short source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(short);
+        nint outputByteStride = outputStride * sizeof(short);
+
+        for (int batch = 0; batch < transformCount; batch += Vector512<short>.Count)
+        {
+            ref byte values = ref Unsafe.As<short, byte>(ref Unsafe.Add(ref source, batch));
+
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+        }
+    }
+
+    /// <summary>
+    /// Applies a packed transform to sixteen independent axes.
+    /// </summary>
+    private static void TransformPackedVector256<TOperator>(
+        Span<short> buffer,
+        int transformCount,
+        int inputStride,
+        int outputStride,
+        int cosBit,
+        Span<int> workspace)
+        where TOperator : struct, IAv1ForwardTransform1dOperator
+    {
+        ref Av1TransformVector<Vector256<short>> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<Vector256<short>>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<Vector256<short>> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref short source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(short);
+        nint outputByteStride = outputStride * sizeof(short);
+
+        for (int batch = 0; batch < transformCount; batch += Vector256<short>.Count)
+        {
+            ref byte values = ref Unsafe.As<short, byte>(ref Unsafe.Add(ref source, batch));
+
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+        }
+    }
+
+    /// <summary>
+    /// Applies a packed transform to eight independent axes.
+    /// </summary>
+    private static void TransformPackedVector128<TOperator>(
+        Span<short> buffer,
+        int transformCount,
+        int inputStride,
+        int outputStride,
+        int cosBit,
+        Span<int> workspace)
+        where TOperator : struct, IAv1ForwardTransform1dOperator
+    {
+        ref Av1TransformVector<Vector128<short>> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<Vector128<short>>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<Vector128<short>> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref short source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(short);
+        nint outputByteStride = outputStride * sizeof(short);
+
+        for (int batch = 0; batch < transformCount; batch += Vector128<short>.Count)
+        {
+            ref byte values = ref Unsafe.As<short, byte>(ref Unsafe.Add(ref source, batch));
+
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+        }
+    }
+
+    /// <summary>
+    /// Applies an expanded transform to sixteen independent axes.
+    /// </summary>
+    private static void TransformExpandedVector512<TOperator>(
+        Span<int> buffer,
+        int transformCount,
+        int inputStride,
+        int outputStride,
+        int cosBit,
+        Span<int> workspace)
+        where TOperator : struct, IAv1ForwardTransform1dOperator
+    {
+        ref Av1TransformVector<Vector512<int>> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<Vector512<int>>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<Vector512<int>> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref int source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(int);
+        nint outputByteStride = outputStride * sizeof(int);
+
+        for (int batch = 0; batch < transformCount; batch += Vector512<int>.Count)
+        {
+            ref byte values = ref Unsafe.As<int, byte>(ref Unsafe.Add(ref source, batch));
+
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+        }
+    }
+
+    /// <summary>
+    /// Applies an expanded transform to eight independent axes.
+    /// </summary>
+    private static void TransformExpandedVector256<TOperator>(
+        Span<int> buffer,
+        int transformCount,
+        int inputStride,
+        int outputStride,
+        int cosBit,
+        Span<int> workspace)
+        where TOperator : struct, IAv1ForwardTransform1dOperator
+    {
+        ref Av1TransformVector<Vector256<int>> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<Vector256<int>>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<Vector256<int>> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref int source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(int);
+        nint outputByteStride = outputStride * sizeof(int);
+
+        for (int batch = 0; batch < transformCount; batch += Vector256<int>.Count)
+        {
+            ref byte values = ref Unsafe.As<int, byte>(ref Unsafe.Add(ref source, batch));
+
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+        }
+    }
+
+    /// <summary>
+    /// Applies an expanded transform to four independent axes.
+    /// </summary>
+    private static void TransformExpandedVector128<TOperator>(
+        Span<int> buffer,
+        int transformCount,
+        int inputStride,
+        int outputStride,
+        int cosBit,
+        Span<int> workspace)
+        where TOperator : struct, IAv1ForwardTransform1dOperator
+    {
+        ref Av1TransformVector<Vector128<int>> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<Vector128<int>>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<Vector128<int>> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref int source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(int);
+        nint outputByteStride = outputStride * sizeof(int);
+
+        for (int batch = 0; batch < transformCount; batch += Vector128<int>.Count)
+        {
+            ref byte values = ref Unsafe.As<int, byte>(ref Unsafe.Add(ref source, batch));
+
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+        }
+    }
+
+    /// <summary>
+    /// Applies an expanded transform to one axis.
+    /// </summary>
+    private static void TransformExpandedScalar<TOperator>(
+        Span<int> buffer,
         int transformCount,
         int inputStride,
         int outputStride,
         int cosBit,
         Span<int> workspace)
         where TOperator : struct, IAv1ForwardTransform1dOperator
-        where TElement : unmanaged
-        where TValue : struct
     {
-        int vectorByteLength = Unsafe.SizeOf<Av1TransformVector<TValue>>();
-        int laneCount = Unsafe.SizeOf<TValue>() / Unsafe.SizeOf<TElement>();
-        ref byte workspaceBase = ref Unsafe.As<int, byte>(ref MemoryMarshal.GetReference(workspace));
-        ref Av1TransformVector<TValue> buffer0 = ref Unsafe.As<byte, Av1TransformVector<TValue>>(ref workspaceBase);
-        ref Av1TransformVector<TValue> buffer1 =
-            ref Unsafe.As<byte, Av1TransformVector<TValue>>(ref Unsafe.Add(ref workspaceBase, vectorByteLength));
-
-        ref TElement sourceBase = ref MemoryMarshal.GetReference(buffer);
-        nint inputByteStride = inputStride * Unsafe.SizeOf<TElement>();
-        nint outputByteStride = outputStride * Unsafe.SizeOf<TElement>();
-
-        // Each lane is an independent row or column. The operators load from and retire coefficients directly to
-        // the strided block, matching Highway's two-buffer stage network without a separate input/output copy pass.
-        for (int batch = 0; batch < transformCount; batch += laneCount)
+        ref Av1TransformVector<int> buffer0 =
+            ref Unsafe.As<int, Av1TransformVector<int>>(ref MemoryMarshal.GetReference(workspace));
+        ref Av1TransformVector<int> buffer1 =
+            ref Unsafe.Add(ref buffer0, 1);
+        ref int source = ref MemoryMarshal.GetReference(buffer);
+        nint inputByteStride = inputStride * sizeof(int);
+        nint outputByteStride = outputStride * sizeof(int);
+
+        for (int batch = 0; batch < transformCount; batch++)
         {
-            ref byte values = ref Unsafe.As<TElement, byte>(ref Unsafe.Add(ref sourceBase, batch));
+            ref byte values = ref Unsafe.As<int, byte>(ref Unsafe.Add(ref source, batch));
 
-            TOperator.Transform<TValue>(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
+            TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit);
         }
     }
 

src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs

@@ -4,7 +4,6 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Runtime.Intrinsics;
-using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
 
 namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
 
@@ -17,7 +16,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
 /// cross-lane permutations. Reconstruction adds the final residuals to their matching prediction lanes before
 /// narrowing to the decoded sample depth.
 /// </remarks>
-internal static class Av1Inverse2dTransformer
+internal static partial class Av1Inverse2dTransformer
 {
     /// <summary>
     /// Applies an inverse transform and adds its residual to high-bit-depth predicted samples.
@@ -39,7 +38,7 @@ internal static class Av1Inverse2dTransformer
         ref Av1Transform2dFlipConfiguration config,
         Span<int> workspace,
         int bitDepth)
-        => Transform2dAdd<short, Av1InverseTransformOutputOperator<short>>(
+        => Transform2dAdd<short, Av1InverseTransformer.HighBitDepthOutputOperator>(
             input,
             outputForRead,
             strideForRead,
@@ -67,7 +66,7 @@ internal static class Av1Inverse2dTransformer
         int strideForWrite,
         ref Av1Transform2dFlipConfiguration config,
         Span<int> workspace)
-        => Transform2dAdd<byte, Av1InverseTransformOutputOperator<byte>>(
+        => Transform2dAdd<byte, Av1InverseTransformer.ByteOutputOperator>(
             input,
             outputForRead,
             strideForRead,
@@ -90,68 +89,68 @@ internal static class Av1Inverse2dTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
     {
         Guard.MustBeSizedAtLeast(workspace, Av1TransformWorkspace.GetRequiredLength(config.TransformSize), nameof(workspace));
         switch (config.TransformFunctionTypeColumn)
         {
             case Av1TransformFunctionType.Dct4:
-                DispatchRow<TSample, TOutputOperator, Av1Dct4Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Dct4Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct8:
-                DispatchRow<TSample, TOutputOperator, Av1Dct8Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Dct8Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct16:
-                DispatchRow<TSample, TOutputOperator, Av1Dct16Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Dct16Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct32:
-                DispatchRow<TSample, TOutputOperator, Av1Dct32Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Dct32Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct64:
-                DispatchRow<TSample, TOutputOperator, Av1Dct64Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Dct64Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Adst4:
-                DispatchRow<TSample, TOutputOperator, Av1Adst4Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Adst4Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Adst8:
-                DispatchRow<TSample, TOutputOperator, Av1Adst8Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Adst8Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Adst16:
-                DispatchRow<TSample, TOutputOperator, Av1Adst16Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Adst16Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity4:
-                DispatchRow<TSample, TOutputOperator, Av1Identity4Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Identity4Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity8:
-                DispatchRow<TSample, TOutputOperator, Av1Identity8Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Identity8Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity16:
-                DispatchRow<TSample, TOutputOperator, Av1Identity16Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Identity16Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity32:
-                DispatchRow<TSample, TOutputOperator, Av1Identity32Inverse1dOperator>(
+                DispatchRow<TSample, TOutputOperator, Identity32Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
@@ -173,68 +172,68 @@ internal static class Av1Inverse2dTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
         where TColumnOperator : struct, IAv1Transform1dOperator
     {
         switch (config.TransformFunctionTypeRow)
         {
             case Av1TransformFunctionType.Dct4:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Dct4Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Dct4Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct8:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Dct8Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Dct8Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct16:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Dct16Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Dct16Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct32:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Dct32Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Dct32Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Dct64:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Dct64Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Dct64Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Adst4:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Adst4Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Adst4Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Adst8:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Adst8Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Adst8Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Adst16:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Adst16Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Adst16Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity4:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Identity4Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Identity4Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity8:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Identity8Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Identity8Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity16:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Identity16Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Identity16Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
             case Av1TransformFunctionType.Identity32:
-                Transform2d<TSample, TOutputOperator, TColumnOperator, Av1Identity32Inverse1dOperator>(
+                Transform2d<TSample, TOutputOperator, TColumnOperator, Identity32Operator>(
                     input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth);
 
                 break;
@@ -256,7 +255,7 @@ internal static class Av1Inverse2dTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
         where TColumnOperator : struct, IAv1Transform1dOperator
         where TRowOperator : struct, IAv1Transform1dOperator
     {
@@ -308,7 +307,7 @@ internal static class Av1Inverse2dTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
         where TColumnOperator : struct, IAv1Transform1dOperator
         where TRowOperator : struct, IAv1Transform1dOperator
     {
@@ -464,7 +463,7 @@ internal static class Av1Inverse2dTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
         where TColumnOperator : struct, IAv1Transform1dOperator
         where TRowOperator : struct, IAv1Transform1dOperator
     {
@@ -596,7 +595,7 @@ internal static class Av1Inverse2dTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
         where TColumnOperator : struct, IAv1Transform1dOperator
         where TRowOperator : struct, IAv1Transform1dOperator
     {

src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs

@@ -1,94 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using SixLabors.ImageSharp.Common.Helpers;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
-
-/// <summary>
-/// Reconstructs AV1 samples from predicted values and inverse-transform residuals.
-/// </summary>
-/// <remarks>
-/// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to
-/// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width
-/// stores. The closed <typeparamref name="TSample"/> specialization removes storage-type branches from hot loops.
-/// </remarks>
-/// <typeparam name="TSample">The decoded sample storage type.</typeparam>
-internal readonly struct Av1InverseTransformOutputOperator<TSample> : IAv1InverseTransformOutputOperator<TSample>
-        where TSample : unmanaged
-{
-    /// <inheritdoc/>
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static TSample Add(TSample prediction, int residual, int bitDepth)
-    {
-        // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each
-        // closed transform so storage selection does not introduce a branch in the reconstruction loop.
-        if (typeof(TSample) == typeof(byte))
-        {
-            byte value = (byte)Math.Clamp(Unsafe.As<TSample, byte>(ref prediction) + residual, byte.MinValue, byte.MaxValue);
-            return Unsafe.As<byte, TSample>(ref value);
-        }
-
-        short result = (short)Math.Clamp(Unsafe.As<TSample, short>(ref prediction) + residual, 0, (1 << bitDepth) - 1);
-        return Unsafe.As<short, TSample>(ref result);
-    }
-
-    /// <inheritdoc/>
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static void Add(ref TSample prediction, ref TSample destination, Vector128<int> residual, int bitDepth)
-    {
-        if (typeof(TSample) == typeof(byte))
-        {
-            // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach
-            // memory, which keeps reconstruction valid at a tightly packed row boundary.
-            ref byte source = ref Unsafe.As<TSample, byte>(ref prediction);
-            uint packed = Unsafe.ReadUnaligned<uint>(ref source);
-            Vector128<ushort> predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
-            Vector128<int> predicted32 = Vector128.WidenLower(predicted16).AsInt32();
-            Vector128<int> reconstructed = Vector128.Clamp(predicted32 + residual, Vector128<int>.Zero, Vector128.Create((int)byte.MaxValue));
-            Vector128<ushort> reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128<uint>.Zero);
-            Vector128<byte> reconstructed8 = Vector128.Narrow(reconstructed16, Vector128<ushort>.Zero);
-            Unsafe.WriteUnaligned(ref Unsafe.As<TSample, byte>(ref destination), reconstructed8.AsUInt32().ToScalar());
-            return;
-        }
-
-        ref short highBitDepthSource = ref Unsafe.As<TSample, short>(ref prediction);
-        ulong highBitDepthPacked = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref highBitDepthSource));
-        Vector128<int> highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16());
-        Vector128<int> highBitDepthReconstructed =
-            Vector128.Clamp(highBitDepthPredicted + residual, Vector128<int>.Zero, Vector128.Create((1 << bitDepth) - 1));
-
-        Vector128<short> narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128<int>.Zero);
-        Unsafe.WriteUnaligned(ref Unsafe.As<TSample, byte>(ref destination), narrowed.AsUInt64().ToScalar());
-    }
-
-    /// <inheritdoc/>
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static void Add(ref TSample prediction, ref TSample destination, Vector256<int> residual, int bitDepth)
-    {
-        if (typeof(TSample) == typeof(byte))
-        {
-            // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store
-            // covers only those reconstructed samples and does not require destination padding.
-            ref byte source = ref Unsafe.As<TSample, byte>(ref prediction);
-            ulong packed = Unsafe.ReadUnaligned<ulong>(ref source);
-            Vector128<ushort> predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
-            Vector256<int> predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32();
-            Vector256<int> reconstructed = Vector256.Clamp(predicted32 + residual, Vector256<int>.Zero, Vector256.Create((int)byte.MaxValue));
-            Vector128<ushort> reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32());
-            Vector128<byte> reconstructed8 = Vector128.Narrow(reconstructed16, Vector128<ushort>.Zero);
-            Unsafe.WriteUnaligned(ref Unsafe.As<TSample, byte>(ref destination), reconstructed8.AsUInt64().ToScalar());
-            return;
-        }
-
-        ref short highBitDepthSource = ref Unsafe.As<TSample, short>(ref prediction);
-        Vector256<int> highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource));
-        Vector256<int> highBitDepthReconstructed =
-            Vector256.Clamp(highBitDepthPredicted + residual, Vector256<int>.Zero, Vector256.Create((1 << bitDepth) - 1));
-
-        Vector128<short> narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper());
-        narrowed.StoreUnsafe(ref Unsafe.As<TSample, short>(ref destination));
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.ByteOutputOperator.cs

@@ -0,0 +1,61 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines eight-bit inverse-transform reconstruction arithmetic.
+/// </content>
+internal partial class Av1InverseTransformer
+{
+    /// <summary>
+    /// Reconstructs eight-bit samples from predicted values and inverse-transform residuals.
+    /// </summary>
+    internal readonly struct ByteOutputOperator : IAv1InverseTransformOutputOperator<byte>
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static byte Add(byte prediction, int residual, int bitDepth)
+        {
+            _ = bitDepth;
+            return (byte)Math.Clamp(prediction + residual, byte.MinValue, byte.MaxValue);
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void Add(ref byte prediction, ref byte destination, Vector128<int> residual, int bitDepth)
+        {
+            _ = bitDepth;
+
+            // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach
+            // memory, which keeps reconstruction valid at a tightly packed row boundary.
+            uint packed = Unsafe.ReadUnaligned<uint>(ref prediction);
+            Vector128<ushort> predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
+            Vector128<int> predicted32 = Vector128.WidenLower(predicted16).AsInt32();
+            Vector128<int> reconstructed = Vector128.Clamp(predicted32 + residual, Vector128<int>.Zero, Vector128.Create((int)byte.MaxValue));
+            Vector128<ushort> reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128<uint>.Zero);
+            Vector128<byte> reconstructed8 = Vector128.Narrow(reconstructed16, Vector128<ushort>.Zero);
+            Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt32().ToScalar());
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void Add(ref byte prediction, ref byte destination, Vector256<int> residual, int bitDepth)
+        {
+            _ = bitDepth;
+
+            // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store
+            // covers only those reconstructed samples and does not require destination padding.
+            ulong packed = Unsafe.ReadUnaligned<ulong>(ref prediction);
+            Vector128<ushort> predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
+            Vector256<int> predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32();
+            Vector256<int> reconstructed = Vector256.Clamp(predicted32 + residual, Vector256<int>.Zero, Vector256.Create((int)byte.MaxValue));
+            Vector128<ushort> reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32());
+            Vector128<byte> reconstructed8 = Vector128.Narrow(reconstructed16, Vector128<ushort>.Zero);
+            Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt64().ToScalar());
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.HighBitDepthOutputOperator.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines high-bit-depth inverse-transform reconstruction arithmetic.
+/// </content>
+internal partial class Av1InverseTransformer
+{
+    /// <summary>
+    /// Reconstructs high-bit-depth samples from predicted values and inverse-transform residuals.
+    /// </summary>
+    internal readonly struct HighBitDepthOutputOperator : IAv1InverseTransformOutputOperator<short>
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static short Add(short prediction, int residual, int bitDepth)
+            => (short)Math.Clamp(prediction + residual, 0, (1 << bitDepth) - 1);
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void Add(ref short prediction, ref short destination, Vector128<int> residual, int bitDepth)
+        {
+            ulong packed = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref prediction));
+            Vector128<int> predicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsInt16());
+            Vector128<int> reconstructed =
+                Vector128.Clamp(predicted + residual, Vector128<int>.Zero, Vector128.Create((1 << bitDepth) - 1));
+
+            Vector128<short> narrowed = Vector128.Narrow(reconstructed, Vector128<int>.Zero);
+            Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref destination), narrowed.AsUInt64().ToScalar());
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void Add(ref short prediction, ref short destination, Vector256<int> residual, int bitDepth)
+        {
+            Vector256<int> predicted = Vector256_.Widen(Vector128.LoadUnsafe(ref prediction));
+            Vector256<int> reconstructed =
+                Vector256.Clamp(predicted + residual, Vector256<int>.Zero, Vector256.Create((1 << bitDepth) - 1));
+
+            Vector128<short> narrowed = Vector128.Narrow(reconstructed.GetLower(), reconstructed.GetUpper());
+            narrowed.StoreUnsafe(ref destination);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the inverse-transform reconstruction operator contract.
+/// </content>
+internal partial class Av1InverseTransformer
+{
+    /// <summary>
+    /// Defines how inverse-transform residuals are added to decoded samples.
+    /// </summary>
+    /// <typeparam name="TSample">The decoded sample storage type.</typeparam>
+    internal interface IAv1InverseTransformOutputOperator<TSample>
+        where TSample : unmanaged
+    {
+        /// <summary>
+        /// Adds one residual to a predicted sample and clips the result to the coded bit depth.
+        /// </summary>
+        /// <param name="prediction">The predicted sample.</param>
+        /// <param name="residual">The inverse-transform residual.</param>
+        /// <param name="bitDepth">The coded sample bit depth.</param>
+        /// <returns>The reconstructed sample.</returns>
+        public static abstract TSample Add(TSample prediction, int residual, int bitDepth);
+
+        /// <summary>
+        /// Adds four residuals to predicted samples and stores the clipped results.
+        /// </summary>
+        /// <param name="prediction">The first predicted sample.</param>
+        /// <param name="destination">The first destination sample.</param>
+        /// <param name="residual">The four inverse-transform residuals.</param>
+        /// <param name="bitDepth">The coded sample bit depth.</param>
+        public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128<int> residual, int bitDepth);
+
+        /// <summary>
+        /// Adds eight residuals to predicted samples and stores the clipped results.
+        /// </summary>
+        /// <param name="prediction">The first predicted sample.</param>
+        /// <param name="destination">The first destination sample.</param>
+        /// <param name="residual">The eight inverse-transform residuals.</param>
+        /// <param name="bitDepth">The coded sample bit depth.</param>
+        public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256<int> residual, int bitDepth);
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs

@@ -6,7 +6,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
 /// <summary>
 /// Reconstructs decoded AV1 transform coefficients into prediction sample buffers.
 /// </summary>
-internal class Av1InverseTransformer
+internal partial class Av1InverseTransformer
 {
     /// <summary>
     /// Reconstructs an eight-bit transform block in place by adding its inverse-transform residual.

src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs

@@ -40,7 +40,7 @@ internal static class Av1InverseWalshHadamardTransformer
         int writeStride,
         int coefficientCount,
         Span<int> workspace)
-        => TransformAdd<byte, Av1InverseTransformOutputOperator<byte>>(
+        => TransformAdd<byte, Av1InverseTransformer.ByteOutputOperator>(
             coefficients,
             readBuffer,
             readStride,
@@ -70,7 +70,7 @@ internal static class Av1InverseWalshHadamardTransformer
         int coefficientCount,
         Span<int> workspace,
         int bitDepth)
-        => TransformAdd<short, Av1InverseTransformOutputOperator<short>>(
+        => TransformAdd<short, Av1InverseTransformer.HighBitDepthOutputOperator>(
             coefficients,
             readBuffer,
             readStride,
@@ -93,7 +93,7 @@ internal static class Av1InverseWalshHadamardTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
     {
         if (Vector128.IsHardwareAccelerated)
         {
@@ -116,7 +116,7 @@ internal static class Av1InverseWalshHadamardTransformer
         int coefficientCount,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
     {
         ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients);
         Vector128<int> row0;
@@ -172,7 +172,7 @@ internal static class Av1InverseWalshHadamardTransformer
         Span<int> workspace,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
     {
         ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer);
         ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer);
@@ -300,7 +300,7 @@ internal static class Av1InverseWalshHadamardTransformer
         Vector128<int> row3,
         int bitDepth)
         where TSample : unmanaged
-        where TOutputOperator : struct, IAv1InverseTransformOutputOperator<TSample>
+        where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator<TSample>
     {
         ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer);
         ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer);

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator.
-/// </summary>
-internal readonly struct Av1Adst16Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the four-point AV1 forward asymmetric discrete sine transform operator.
-/// </summary>
-internal readonly struct Av1Adst4Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the eight-point AV1 forward asymmetric discrete sine transform operator.
-/// </summary>
-internal readonly struct Av1Adst8Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the sixteen-point AV1 forward discrete cosine transform operator.
-/// </summary>
-internal readonly struct Av1Dct16Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the thirty-two-point AV1 forward discrete cosine transform operator.
-/// </summary>
-internal readonly struct Av1Dct32Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the four-point AV1 forward discrete cosine transform operator.
-/// </summary>
-internal readonly struct Av1Dct4Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the sixty-four-point AV1 forward discrete cosine transform operator.
-/// </summary>
-internal readonly struct Av1Dct64Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the eight-point AV1 forward discrete cosine transform operator.
-/// </summary>
-internal readonly struct Av1Dct8Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs

@@ -1,314 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <content>
-/// Implements the forward asymmetric discrete sine transform stage networks.
-/// </content>
-internal static partial class Av1ForwardTransformOperations
-{
-    /// <summary>
-    /// Gets the fixed eight-point ADST coefficient permutation.
-    /// </summary>
-    private static ReadOnlySpan<byte> Adst8OutputOrder => [1, 6, 3, 4, 5, 2, 7, 0];
-
-    /// <summary>
-    /// Gets the first cosine index for each final sixteen-point ADST rotation.
-    /// </summary>
-    private static ReadOnlySpan<byte> Adst16FinalWeights => [2, 10, 18, 26, 34, 42, 50, 58];
-
-    /// <summary>
-    /// Gets the fixed sixteen-point ADST coefficient permutation.
-    /// </summary>
-    private static ReadOnlySpan<byte> Adst16OutputOrder => [1, 14, 3, 12, 5, 10, 7, 8, 9, 6, 11, 4, 13, 2, 15, 0];
-
-    /// <summary>
-    /// Applies the four-point forward asymmetric discrete sine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The unused first transform-stage buffer.</param>
-    /// <param name="buffer1">The unused second transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the sine constants.</param>
-    public static void Adst4<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        _ = buffer0;
-        _ = buffer1;
-
-        ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-        TValue input0 = Load<TValue>(ref values, inputStride, 0);
-        TValue input1 = Load<TValue>(ref values, inputStride, 1);
-        TValue input2 = Load<TValue>(ref values, inputStride, 2);
-        TValue input3 = Load<TValue>(ref values, inputStride, 3);
-        TValue input01 = Av1ForwardTransformArithmetic<TValue>.Add(input0, input1);
-
-        // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
-        TValue output0 = Av1ForwardTransformArithmetic<TValue>.MultiplyAddRound(
-            sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
-
-        TValue output1 = Av1ForwardTransformArithmetic<TValue>.MultiplyAddRound(
-            sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
-
-        TValue output2 = Av1ForwardTransformArithmetic<TValue>.MultiplyAddRound(
-            sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
-
-        // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
-        TValue output3 = Av1ForwardTransformArithmetic<TValue>.MultiplyAddRound(
-            sinpi[4] - sinpi[1],
-            input0,
-            -sinpi[1] - sinpi[2],
-            input1,
-            sinpi[3],
-            input2,
-            sinpi[2] - sinpi[4],
-            input3,
-            cosBit,
-            in rounding);
-
-        Store(ref values, outputStride, 0, output0);
-        Store(ref values, outputStride, 1, output1);
-        Store(ref values, outputStride, 2, output2);
-        Store(ref values, outputStride, 3, output3);
-    }
-
-    /// <summary>
-    /// Applies the eight-point forward asymmetric discrete sine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Adst8<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
-        buffer0[0] = Load<TValue>(ref values, inputStride, 0);
-        buffer0[1] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 7));
-        buffer0[2] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 3));
-        buffer0[3] = Load<TValue>(ref values, inputStride, 4);
-        buffer0[4] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 1));
-        buffer0[5] = Load<TValue>(ref values, inputStride, 6);
-        buffer0[6] = Load<TValue>(ref values, inputStride, 2);
-        buffer0[7] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 5));
-
-        // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
-        buffer1[0] = buffer0[0];
-        buffer1[1] = buffer0[1];
-        Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
-        buffer1[4] = buffer0[4];
-        buffer1[5] = buffer0[5];
-        Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
-
-        // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
-        for (int group = 0; group < 8; group += 4)
-        {
-            for (int i = 0; i < 2; i++)
-            {
-                Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                    buffer1[group + i],
-                    buffer1[group + i + 2],
-                    out buffer0[group + i],
-                    out buffer0[group + i + 2]);
-            }
-        }
-
-        // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
-        for (int i = 0; i < 4; i++)
-        {
-            buffer1[i] = buffer0[i];
-        }
-
-        buffer1[4] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
-        buffer1[5] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
-        buffer1[6] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
-        buffer1[7] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
-
-        // Stage 5 creates the four final butterfly pairs spanning the two groups.
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
-        }
-
-        // Stage 6 applies the remaining odd-angle rotations.
-        buffer1[0] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
-        buffer1[1] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
-        buffer1[2] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
-        buffer1[3] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
-        buffer1[4] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
-        buffer1[5] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
-        buffer1[6] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
-        buffer1[7] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
-
-        ReadOnlySpan<byte> outputOrder = Adst8OutputOrder;
-
-        // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
-        for (int i = 0; i < 8; i++)
-        {
-            Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
-        }
-    }
-
-    /// <summary>
-    /// Applies the sixteen-point forward asymmetric discrete sine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Adst16<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        // Stage 1 is the bit-reversed ADST input order with the normative alternating signs.
-        buffer0[0] = Load<TValue>(ref values, inputStride, 0);
-        buffer0[1] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 15));
-        buffer0[2] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 7));
-        buffer0[3] = Load<TValue>(ref values, inputStride, 8);
-        buffer0[4] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 3));
-        buffer0[5] = Load<TValue>(ref values, inputStride, 12);
-        buffer0[6] = Load<TValue>(ref values, inputStride, 4);
-        buffer0[7] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 11));
-        buffer0[8] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 1));
-        buffer0[9] = Load<TValue>(ref values, inputStride, 14);
-        buffer0[10] = Load<TValue>(ref values, inputStride, 6);
-        buffer0[11] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 9));
-        buffer0[12] = Load<TValue>(ref values, inputStride, 2);
-        buffer0[13] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 13));
-        buffer0[14] = Av1ForwardTransformArithmetic<TValue>.Negate(Load<TValue>(ref values, inputStride, 5));
-        buffer0[15] = Load<TValue>(ref values, inputStride, 10);
-
-        // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged.
-        for (int group = 0; group < 16; group += 4)
-        {
-            buffer1[group] = buffer0[group];
-            buffer1[group + 1] = buffer0[group + 1];
-            Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding);
-        }
-
-        // Stage 3 combines adjacent pairs within each group of four.
-        for (int group = 0; group < 16; group += 4)
-        {
-            for (int i = 0; i < 2; i++)
-            {
-                Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                    buffer1[group + i],
-                    buffer1[group + i + 2],
-                    out buffer0[group + i],
-                    out buffer0[group + i + 2]);
-            }
-        }
-
-        // Stage 4 rotates the upper pair of each eight-value group by pi/8.
-        for (int group = 0; group < 16; group += 8)
-        {
-            for (int i = 0; i < 4; i++)
-            {
-                buffer1[group + i] = buffer0[group + i];
-            }
-
-            buffer1[group + 4] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(
-                cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding);
-
-            buffer1[group + 5] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(
-                cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding);
-
-            buffer1[group + 6] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(
-                -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding);
-
-            buffer1[group + 7] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(
-                cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding);
-        }
-
-        // Stage 5 combines the lower and upper quartets within each eight-value group.
-        for (int group = 0; group < 16; group += 8)
-        {
-            for (int i = 0; i < 4; i++)
-            {
-                Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                    buffer1[group + i],
-                    buffer1[group + i + 4],
-                    out buffer0[group + i],
-                    out buffer0[group + i + 4]);
-            }
-        }
-
-        // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet.
-        for (int i = 0; i < 8; i++)
-        {
-            buffer1[i] = buffer0[i];
-        }
-
-        buffer1[8] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding);
-        buffer1[9] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding);
-        buffer1[10] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding);
-        buffer1[11] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding);
-        buffer1[12] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding);
-        buffer1[13] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding);
-        buffer1[14] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding);
-        buffer1[15] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding);
-
-        // Stage 7 creates the eight final butterfly pairs spanning both octets.
-        for (int i = 0; i < 8; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]);
-        }
-
-        ReadOnlySpan<byte> finalWeights = Adst16FinalWeights;
-
-        // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order
-        // without allocating a per-call array or duplicating the complementary cosine-index calculation.
-        for (int pair = 0; pair < 8; pair++)
-        {
-            int first = finalWeights[pair];
-            int second = 64 - first;
-            int index = pair * 2;
-            buffer1[index] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(
-                cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding);
-
-            buffer1[index + 1] = Av1ForwardTransformArithmetic<TValue>.HalfButterfly(
-                cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding);
-        }
-
-        ReadOnlySpan<byte> outputOrder = Adst16OutputOrder;
-
-        // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order.
-        for (int i = 0; i < 16; i++)
-        {
-            Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
-        }
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs

@@ -1,218 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <content>
-/// Implements the sixteen-point forward DCT stage network.
-/// </content>
-internal static partial class Av1ForwardTransformOperations
-{
-    /// <summary>
-    /// Applies the sixteen-point forward discrete cosine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Dct16<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations.
-        for (int i = 0; i < 8; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                Load<TValue>(ref values, inputStride, i),
-                Load<TValue>(ref values, inputStride, 15 - i),
-                out buffer0[i],
-                out buffer0[15 - i]);
-        }
-
-        // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4.
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]);
-        }
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer0[10],
-            buffer0[13],
-            out buffer1[10],
-            out buffer1[13],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer0[11],
-            buffer0[12],
-            out buffer1[11],
-            out buffer1[12],
-            cosBit,
-            in rounding);
-
-        // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations.
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]);
-        }
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer1[5],
-            buffer1[6],
-            out buffer0[5],
-            out buffer0[6],
-            cosBit,
-            in rounding);
-
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]);
-        }
-
-        // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order.
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[32],
-            cospi[32],
-            buffer0[0],
-            buffer0[1],
-            out TValue output0,
-            out TValue output8,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[16],
-            cospi[48],
-            buffer0[3],
-            buffer0[2],
-            out TValue output4,
-            out TValue output12,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]);
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[16],
-            cospi[48],
-            buffer0[9],
-            buffer0[14],
-            out buffer1[9],
-            out buffer1[14],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[48],
-            -cospi[16],
-            buffer0[10],
-            buffer0[13],
-            out buffer1[10],
-            out buffer1[13],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[8],
-            cospi[56],
-            buffer1[7],
-            buffer1[4],
-            out TValue output2,
-            out TValue output14,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[40],
-            cospi[24],
-            buffer1[6],
-            buffer1[5],
-            out TValue output10,
-            out TValue output6,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]);
-
-        // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative
-        // coefficient permutation, so each rotation result is named by its final destination.
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[4],
-            cospi[60],
-            buffer0[15],
-            buffer0[8],
-            out TValue output1,
-            out TValue output15,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[36],
-            cospi[28],
-            buffer0[14],
-            buffer0[9],
-            out TValue output9,
-            out TValue output7,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[20],
-            cospi[44],
-            buffer0[13],
-            buffer0[10],
-            out TValue output5,
-            out TValue output11,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[52],
-            cospi[12],
-            buffer0[12],
-            buffer0[11],
-            out TValue output13,
-            out TValue output3,
-            cosBit,
-            in rounding);
-
-        Store(ref values, outputStride, 0, output0);
-        Store(ref values, outputStride, 1, output1);
-        Store(ref values, outputStride, 2, output2);
-        Store(ref values, outputStride, 3, output3);
-        Store(ref values, outputStride, 4, output4);
-        Store(ref values, outputStride, 5, output5);
-        Store(ref values, outputStride, 6, output6);
-        Store(ref values, outputStride, 7, output7);
-        Store(ref values, outputStride, 8, output8);
-        Store(ref values, outputStride, 9, output9);
-        Store(ref values, outputStride, 10, output10);
-        Store(ref values, outputStride, 11, output11);
-        Store(ref values, outputStride, 12, output12);
-        Store(ref values, outputStride, 13, output13);
-        Store(ref values, outputStride, 14, output14);
-        Store(ref values, outputStride, 15, output15);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs

@@ -1,263 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <content>
-/// Implements the thirty-two-point forward DCT stage network.
-/// </content>
-internal static partial class Av1ForwardTransformOperations
-{
-    /// <summary>
-    /// Applies the thirty-two-point forward discrete cosine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Dct32<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        // Stage 1 consumes the source block completely before any final coefficient is stored back into it.
-        for (int i = 0; i < 16; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                Load<TValue>(ref values, inputStride, i),
-                Load<TValue>(ref values, inputStride, 31 - i),
-                out buffer1[i],
-                out buffer1[31 - i]);
-        }
-
-        // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs.
-        for (int i = 0; i < 8; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]);
-        }
-
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.Butterfly(
-                -cospi[32],
-                cospi[32],
-                buffer1[20 + i],
-                buffer1[27 - i],
-                out buffer0[20 + i],
-                out buffer0[27 - i],
-                cosBit,
-                in rounding);
-        }
-
-        // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences.
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]);
-        }
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer0[10],
-            buffer0[13],
-            out buffer1[10],
-            out buffer1[13],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer0[11],
-            buffer0[12],
-            out buffer1[11],
-            out buffer1[12],
-            cosBit,
-            in rounding);
-
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]);
-        }
-
-        // Stage 4 continues the factorization as independent eight-value groups.
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]);
-        }
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer1[5],
-            buffer1[6],
-            out buffer0[5],
-            out buffer0[6],
-            cosBit,
-            in rounding);
-
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]);
-        }
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[16],
-            cospi[48],
-            buffer1[18],
-            buffer1[29],
-            out buffer0[18],
-            out buffer0[29],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[16],
-            cospi[48],
-            buffer1[19],
-            buffer1[28],
-            out buffer0[19],
-            out buffer0[28],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[48],
-            -cospi[16],
-            buffer1[20],
-            buffer1[27],
-            out buffer0[20],
-            out buffer0[27],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[48],
-            -cospi[16],
-            buffer1[21],
-            buffer1[26],
-            out buffer0[21],
-            out buffer0[26],
-            cosBit,
-            in rounding);
-
-        // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are
-        // retired directly to the block instead of being copied through a third workspace.
-        ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding);
-        ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]);
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[16],
-            cospi[48],
-            buffer0[9],
-            buffer0[14],
-            out buffer1[9],
-            out buffer1[14],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[48],
-            -cospi[16],
-            buffer0[10],
-            buffer0[13],
-            out buffer1[10],
-            out buffer1[13],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]);
-
-        // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations.
-        ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding);
-        ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]);
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[8],
-            cospi[56],
-            buffer1[17],
-            buffer1[30],
-            out buffer0[17],
-            out buffer0[30],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[56],
-            -cospi[8],
-            buffer1[18],
-            buffer1[29],
-            out buffer0[18],
-            out buffer0[29],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[40],
-            cospi[24],
-            buffer1[21],
-            buffer1[26],
-            out buffer0[21],
-            out buffer0[26],
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[24],
-            -cospi[40],
-            buffer1[22],
-            buffer1[25],
-            out buffer0[22],
-            out buffer0[25],
-            cosBit,
-            in rounding);
-
-        // Stage 7 applies the pi/32 rotations to the next odd-frequency level.
-        ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding);
-        ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding);
-        ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding);
-        ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]);
-
-        // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results
-        // are consumed by another arithmetic stage.
-        ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding);
-        ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding);
-        ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding);
-        ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding);
-        ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding);
-        ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding);
-        ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding);
-        ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs

@@ -1,285 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <content>
-/// Implements the sixty-four-point forward DCT stage network.
-/// </content>
-internal static partial class Av1ForwardTransformOperations
-{
-    /// <summary>
-    /// Identifies coefficient positions whose final value resides in the first stage buffer.
-    /// </summary>
-    private const ulong Dct64Buffer0OutputMask =
-        (1UL << 2) | (1UL << 6) | (1UL << 8) | (1UL << 10) | (1UL << 14) |
-        (1UL << 18) | (1UL << 22) | (1UL << 24) | (1UL << 26) | (1UL << 30) |
-        (1UL << 34) | (1UL << 38) | (1UL << 40) | (1UL << 42) | (1UL << 46) |
-        (1UL << 50) | (1UL << 54) | (1UL << 56) | (1UL << 58) | (1UL << 62);
-
-    /// <summary>
-    /// Gets the stage-nine rotation order for the middle quarter of the sixty-four-point DCT.
-    /// </summary>
-    private static ReadOnlySpan<byte> Dct64Stage9RotationOrder => [2, 34, 18, 50, 10, 42, 26, 58];
-
-    /// <summary>
-    /// Gets the stage-ten rotation order for the upper half of the sixty-four-point DCT.
-    /// </summary>
-    private static ReadOnlySpan<byte> Dct64Stage10RotationOrder => [1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61];
-
-    /// <summary>
-    /// Gets the mapping from coefficient order to the final staged value.
-    /// </summary>
-    private static ReadOnlySpan<byte> Dct64OutputOrder =>
-    [
-        0, 32, 16, 48, 8, 40, 24, 56, 4, 36, 20, 52, 12, 44, 28, 60,
-        2, 34, 18, 50, 10, 42, 26, 58, 6, 38, 22, 54, 14, 46, 30, 62,
-        1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61,
-        3, 35, 19, 51, 11, 43, 27, 59, 7, 39, 23, 55, 15, 47, 31, 63,
-    ];
-
-    /// <summary>
-    /// Applies the sixty-four-point forward discrete cosine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Dct64<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients.
-        for (int i = 0; i < 32; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                Load<TValue>(ref values, inputStride, i),
-                Load<TValue>(ref values, inputStride, 63 - i),
-                out buffer0[i],
-                out buffer0[63 - i]);
-        }
-
-        // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs.
-        for (int i = 0; i < 16; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]);
-        }
-
-        for (int i = 0; i < 8; i++)
-        {
-            Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding);
-        }
-
-        // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences.
-        for (int i = 0; i < 8; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]);
-        }
-
-        for (int i = 0; i < 4; i++)
-        {
-            Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding);
-        }
-
-        for (int i = 0; i < 8; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]);
-        }
-
-        for (int i = 0; i < 8; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]);
-        }
-
-        // Stage 4 continues the factorization as independent sixteen-value groups.
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]);
-        }
-
-        for (int i = 0; i < 2; i++)
-        {
-            Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding);
-        }
-
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]);
-            Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding);
-        }
-
-        for (int i = 4; i < 8; i++)
-        {
-            Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding);
-        }
-
-        // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks.
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]);
-        }
-
-        Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding);
-
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]);
-            Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding);
-        }
-
-        for (int i = 2; i < 4; i++)
-        {
-            Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding);
-        }
-
-        for (int i = 0; i < 4; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]);
-        }
-
-        // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups.
-        Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding);
-        Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]);
-        Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding);
-        Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding);
-
-        for (int i = 0; i < 2; i++)
-        {
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]);
-            Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]);
-            Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding);
-            Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding);
-        }
-
-        for (int i = 2; i < 4; i++)
-        {
-            Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding);
-            Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding);
-        }
-
-        // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations.
-        Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding);
-        Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]);
-        Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding);
-        Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding);
-        Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding);
-        Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding);
-
-        for (int group = 0; group < 4; group++)
-        {
-            int offset = group * 8;
-
-            for (int i = 0; i < 2; i++)
-            {
-                Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                    buffer0[32 + offset + i],
-                    buffer1[35 + offset - i],
-                    out buffer0[32 + offset + i],
-                    out buffer0[35 + offset - i]);
-
-                Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-                    buffer0[39 + offset - i],
-                    buffer1[36 + offset + i],
-                    out buffer0[39 + offset - i],
-                    out buffer0[36 + offset + i]);
-            }
-        }
-
-        // Stage 8 applies the next level of odd-frequency rotations.
-        Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding);
-        Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding);
-        Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding);
-        Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]);
-
-        Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding);
-        Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding);
-        Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding);
-        Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding);
-        Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding);
-        Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding);
-        Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding);
-        Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding);
-
-        // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the
-        // non-linear rotation order while keeping the constants in compile-time data.
-        for (int i = 0; i < 8; i++)
-        {
-            int low = 16 + i;
-            int high = 31 - i;
-            int odd = Dct64Stage9RotationOrder[i];
-            Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding);
-        }
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]);
-
-        // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level.
-        for (int i = 0; i < 16; i++)
-        {
-            int low = 32 + i;
-            int high = 63 - i;
-            int odd = Dct64Stage10RotationOrder[i];
-            Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding);
-        }
-
-        // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and
-        // 16-31 remain in buffer0 at retirement,
-        // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order.
-        ReadOnlySpan<byte> outputOrder = Dct64OutputOrder;
-
-        for (int i = 0; i < 64; i++)
-        {
-            int sourceIndex = outputOrder[i];
-            TValue value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex];
-
-            Store(ref values, outputStride, i, value);
-        }
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs

@@ -1,125 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <content>
-/// Implements the eight-point forward DCT stage network.
-/// </content>
-internal static partial class Av1ForwardTransformOperations
-{
-    /// <summary>
-    /// Applies the eight-point forward discrete cosine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Dct8<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
-        // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-            Load<TValue>(ref values, inputStride, 0),
-            Load<TValue>(ref values, inputStride, 7),
-            out buffer0[0],
-            out buffer1[7]);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-            Load<TValue>(ref values, inputStride, 1),
-            Load<TValue>(ref values, inputStride, 6),
-            out buffer0[1],
-            out buffer0[6]);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-            Load<TValue>(ref values, inputStride, 2),
-            Load<TValue>(ref values, inputStride, 5),
-            out buffer0[2],
-            out buffer0[5]);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(
-            Load<TValue>(ref values, inputStride, 3),
-            Load<TValue>(ref values, inputStride, 4),
-            out buffer0[3],
-            out buffer1[4]);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            -cospi[32],
-            cospi[32],
-            buffer0[5],
-            buffer0[6],
-            out buffer1[5],
-            out buffer1[6],
-            cosBit,
-            in rounding);
-
-        // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[32],
-            cospi[32],
-            buffer1[0],
-            buffer1[1],
-            out TValue output0,
-            out TValue output4,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[16],
-            cospi[48],
-            buffer1[3],
-            buffer1[2],
-            out TValue output2,
-            out TValue output6,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
-
-        // Highway fuses the final two stages because no intermediate value is reused after either rotation.
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[8],
-            cospi[56],
-            buffer0[7],
-            buffer0[4],
-            out TValue output1,
-            out TValue output7,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[40],
-            cospi[24],
-            buffer0[6],
-            buffer0[5],
-            out TValue output5,
-            out TValue output3,
-            cosBit,
-            in rounding);
-
-        Store(ref values, outputStride, 0, output0);
-        Store(ref values, outputStride, 1, output1);
-        Store(ref values, outputStride, 2, output2);
-        Store(ref values, outputStride, 3, output3);
-        Store(ref values, outputStride, 4, output4);
-        Store(ref values, outputStride, 5, output5);
-        Store(ref values, outputStride, 6, output6);
-        Store(ref values, outputStride, 7, output7);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs

@@ -1,135 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <content>
-/// Implements the length-specific forward identity transform scaling.
-/// </content>
-internal static partial class Av1ForwardTransformOperations
-{
-    /// <summary>
-    /// Applies the four-point forward identity transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Identity4<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 4, 1, 0);
-
-    /// <summary>
-    /// Applies the eight-point forward identity transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Identity8<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 8, 0, 1);
-
-    /// <summary>
-    /// Applies the sixteen-point forward identity transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Identity16<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 16, 2, 0);
-
-    /// <summary>
-    /// Applies the thirty-two-point forward identity transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Identity32<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 32, 0, 2);
-
-    /// <summary>
-    /// Applies the length-specific AV1 identity scaling directly to the strided transform block.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="length">The number of transform positions.</param>
-    /// <param name="sqrt2Scale">The square-root-of-two multiplier, or zero when power-of-two scaling applies.</param>
-    /// <param name="leftShift">The power-of-two scaling shift.</param>
-    private static void Identity<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit,
-        int length,
-        int sqrt2Scale,
-        int leftShift)
-        where TValue : struct
-    {
-        _ = buffer0;
-        _ = buffer1;
-        _ = cosBit;
-
-        // AV1 defines identity normalization by transform length: 4 and 16 use sqrt(2) scaling, while 8 and 32
-        // are exact powers of two. Applying it in place matches Highway's row-oriented identity kernels.
-        for (int i = 0; i < length; i++)
-        {
-            TValue input = Load<TValue>(ref values, inputStride, i);
-            TValue output = sqrt2Scale != 0
-                ? Av1ForwardTransformArithmetic<TValue>.MultiplyRound(
-                    input,
-                    sqrt2Scale * Av1Transform1dMath.NewSqrt2,
-                    Av1Transform1dMath.NewSqrt2Bits)
-                : Av1ForwardTransformArithmetic<TValue>.ShiftLeft(input, leftShift);
-
-            Store(ref values, outputStride, i, output);
-        }
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst4Operator.cs

@@ -0,0 +1,411 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the four-point forward ADST operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the four-point forward asymmetric discrete sine transform.
+    /// </summary>
+    internal readonly struct Adst4Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<int>.CreateRounding(cosBit);
+            int input0 = Load<int>(ref values, inputStride, 0);
+            int input1 = Load<int>(ref values, inputStride, 1);
+            int input2 = Load<int>(ref values, inputStride, 2);
+            int input3 = Load<int>(ref values, inputStride, 3);
+            int input01 = Av1ForwardTransformArithmetic<int>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            int output0 = Av1ForwardTransformArithmetic<int>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            int output1 = Av1ForwardTransformArithmetic<int>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            int output2 = Av1ForwardTransformArithmetic<int>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            int output3 = Av1ForwardTransformArithmetic<int>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<short>.CreateRounding(cosBit);
+            short input0 = Load<short>(ref values, inputStride, 0);
+            short input1 = Load<short>(ref values, inputStride, 1);
+            short input2 = Load<short>(ref values, inputStride, 2);
+            short input3 = Load<short>(ref values, inputStride, 3);
+            short input01 = Av1ForwardTransformArithmetic<short>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            short output0 = Av1ForwardTransformArithmetic<short>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            short output1 = Av1ForwardTransformArithmetic<short>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            short output2 = Av1ForwardTransformArithmetic<short>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            short output3 = Av1ForwardTransformArithmetic<short>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<short>>.CreateRounding(cosBit);
+            Vector128<short> input0 = Load<Vector128<short>>(ref values, inputStride, 0);
+            Vector128<short> input1 = Load<Vector128<short>>(ref values, inputStride, 1);
+            Vector128<short> input2 = Load<Vector128<short>>(ref values, inputStride, 2);
+            Vector128<short> input3 = Load<Vector128<short>>(ref values, inputStride, 3);
+            Vector128<short> input01 = Av1ForwardTransformArithmetic<Vector128<short>>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            Vector128<short> output0 = Av1ForwardTransformArithmetic<Vector128<short>>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            Vector128<short> output1 = Av1ForwardTransformArithmetic<Vector128<short>>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            Vector128<short> output2 = Av1ForwardTransformArithmetic<Vector128<short>>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            Vector128<short> output3 = Av1ForwardTransformArithmetic<Vector128<short>>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<short>>.CreateRounding(cosBit);
+            Vector256<short> input0 = Load<Vector256<short>>(ref values, inputStride, 0);
+            Vector256<short> input1 = Load<Vector256<short>>(ref values, inputStride, 1);
+            Vector256<short> input2 = Load<Vector256<short>>(ref values, inputStride, 2);
+            Vector256<short> input3 = Load<Vector256<short>>(ref values, inputStride, 3);
+            Vector256<short> input01 = Av1ForwardTransformArithmetic<Vector256<short>>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            Vector256<short> output0 = Av1ForwardTransformArithmetic<Vector256<short>>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            Vector256<short> output1 = Av1ForwardTransformArithmetic<Vector256<short>>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            Vector256<short> output2 = Av1ForwardTransformArithmetic<Vector256<short>>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            Vector256<short> output3 = Av1ForwardTransformArithmetic<Vector256<short>>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<short>>.CreateRounding(cosBit);
+            Vector512<short> input0 = Load<Vector512<short>>(ref values, inputStride, 0);
+            Vector512<short> input1 = Load<Vector512<short>>(ref values, inputStride, 1);
+            Vector512<short> input2 = Load<Vector512<short>>(ref values, inputStride, 2);
+            Vector512<short> input3 = Load<Vector512<short>>(ref values, inputStride, 3);
+            Vector512<short> input01 = Av1ForwardTransformArithmetic<Vector512<short>>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            Vector512<short> output0 = Av1ForwardTransformArithmetic<Vector512<short>>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            Vector512<short> output1 = Av1ForwardTransformArithmetic<Vector512<short>>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            Vector512<short> output2 = Av1ForwardTransformArithmetic<Vector512<short>>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            Vector512<short> output3 = Av1ForwardTransformArithmetic<Vector512<short>>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<int>>.CreateRounding(cosBit);
+            Vector128<int> input0 = Load<Vector128<int>>(ref values, inputStride, 0);
+            Vector128<int> input1 = Load<Vector128<int>>(ref values, inputStride, 1);
+            Vector128<int> input2 = Load<Vector128<int>>(ref values, inputStride, 2);
+            Vector128<int> input3 = Load<Vector128<int>>(ref values, inputStride, 3);
+            Vector128<int> input01 = Av1ForwardTransformArithmetic<Vector128<int>>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            Vector128<int> output0 = Av1ForwardTransformArithmetic<Vector128<int>>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            Vector128<int> output1 = Av1ForwardTransformArithmetic<Vector128<int>>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            Vector128<int> output2 = Av1ForwardTransformArithmetic<Vector128<int>>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            Vector128<int> output3 = Av1ForwardTransformArithmetic<Vector128<int>>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<int>>.CreateRounding(cosBit);
+            Vector256<int> input0 = Load<Vector256<int>>(ref values, inputStride, 0);
+            Vector256<int> input1 = Load<Vector256<int>>(ref values, inputStride, 1);
+            Vector256<int> input2 = Load<Vector256<int>>(ref values, inputStride, 2);
+            Vector256<int> input3 = Load<Vector256<int>>(ref values, inputStride, 3);
+            Vector256<int> input01 = Av1ForwardTransformArithmetic<Vector256<int>>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            Vector256<int> output0 = Av1ForwardTransformArithmetic<Vector256<int>>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            Vector256<int> output1 = Av1ForwardTransformArithmetic<Vector256<int>>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            Vector256<int> output2 = Av1ForwardTransformArithmetic<Vector256<int>>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            Vector256<int> output3 = Av1ForwardTransformArithmetic<Vector256<int>>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<int>>.CreateRounding(cosBit);
+            Vector512<int> input0 = Load<Vector512<int>>(ref values, inputStride, 0);
+            Vector512<int> input1 = Load<Vector512<int>>(ref values, inputStride, 1);
+            Vector512<int> input2 = Load<Vector512<int>>(ref values, inputStride, 2);
+            Vector512<int> input3 = Load<Vector512<int>>(ref values, inputStride, 3);
+            Vector512<int> input01 = Av1ForwardTransformArithmetic<Vector512<int>>.Add(input0, input1);
+
+            // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic.
+            Vector512<int> output0 = Av1ForwardTransformArithmetic<Vector512<int>>.MultiplyAddRound(
+                sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding);
+
+            Vector512<int> output1 = Av1ForwardTransformArithmetic<Vector512<int>>.MultiplyAddRound(
+                sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding);
+
+            Vector512<int> output2 = Av1ForwardTransformArithmetic<Vector512<int>>.MultiplyAddRound(
+                sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding);
+
+            // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point.
+            Vector512<int> output3 = Av1ForwardTransformArithmetic<Vector512<int>>.MultiplyAddRound(
+                sinpi[4] - sinpi[1],
+                input0,
+                -sinpi[1] - sinpi[2],
+                input1,
+                sinpi[3],
+                input2,
+                sinpi[2] - sinpi[4],
+                input3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst8Operator.cs

@@ -0,0 +1,656 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the eight-point forward ADST operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the eight-point forward asymmetric discrete sine transform.
+    /// </summary>
+    internal readonly struct Adst8Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <summary>
+        /// Gets the fixed coefficient permutation.
+        /// </summary>
+        private static ReadOnlySpan<byte> OutputOrder => [1, 6, 3, 4, 5, 2, 7, 0];
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<int>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<int>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<int>.Negate(Load<int>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<int>.Negate(Load<int>(ref values, inputStride, 3));
+            buffer0[3] = Load<int>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<int>.Negate(Load<int>(ref values, inputStride, 1));
+            buffer0[5] = Load<int>(ref values, inputStride, 6);
+            buffer0[6] = Load<int>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<int>.Negate(Load<int>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<int>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<int>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<int>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<int>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<short>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<short>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<short>.Negate(Load<short>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<short>.Negate(Load<short>(ref values, inputStride, 3));
+            buffer0[3] = Load<short>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<short>.Negate(Load<short>(ref values, inputStride, 1));
+            buffer0[5] = Load<short>(ref values, inputStride, 6);
+            buffer0[6] = Load<short>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<short>.Negate(Load<short>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<short>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<short>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<short>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<short>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<short>>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<Vector128<short>>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<Vector128<short>>.Negate(Load<Vector128<short>>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<Vector128<short>>.Negate(Load<Vector128<short>>(ref values, inputStride, 3));
+            buffer0[3] = Load<Vector128<short>>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<Vector128<short>>.Negate(Load<Vector128<short>>(ref values, inputStride, 1));
+            buffer0[5] = Load<Vector128<short>>(ref values, inputStride, 6);
+            buffer0[6] = Load<Vector128<short>>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<Vector128<short>>.Negate(Load<Vector128<short>>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector128<short>>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<short>>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<Vector256<short>>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<Vector256<short>>.Negate(Load<Vector256<short>>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<Vector256<short>>.Negate(Load<Vector256<short>>(ref values, inputStride, 3));
+            buffer0[3] = Load<Vector256<short>>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<Vector256<short>>.Negate(Load<Vector256<short>>(ref values, inputStride, 1));
+            buffer0[5] = Load<Vector256<short>>(ref values, inputStride, 6);
+            buffer0[6] = Load<Vector256<short>>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<Vector256<short>>.Negate(Load<Vector256<short>>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector256<short>>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<short>>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<Vector512<short>>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<Vector512<short>>.Negate(Load<Vector512<short>>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<Vector512<short>>.Negate(Load<Vector512<short>>(ref values, inputStride, 3));
+            buffer0[3] = Load<Vector512<short>>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<Vector512<short>>.Negate(Load<Vector512<short>>(ref values, inputStride, 1));
+            buffer0[5] = Load<Vector512<short>>(ref values, inputStride, 6);
+            buffer0[6] = Load<Vector512<short>>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<Vector512<short>>.Negate(Load<Vector512<short>>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector512<short>>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<int>>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<Vector128<int>>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<Vector128<int>>.Negate(Load<Vector128<int>>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<Vector128<int>>.Negate(Load<Vector128<int>>(ref values, inputStride, 3));
+            buffer0[3] = Load<Vector128<int>>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<Vector128<int>>.Negate(Load<Vector128<int>>(ref values, inputStride, 1));
+            buffer0[5] = Load<Vector128<int>>(ref values, inputStride, 6);
+            buffer0[6] = Load<Vector128<int>>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<Vector128<int>>.Negate(Load<Vector128<int>>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector128<int>>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<int>>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<Vector256<int>>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<Vector256<int>>.Negate(Load<Vector256<int>>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<Vector256<int>>.Negate(Load<Vector256<int>>(ref values, inputStride, 3));
+            buffer0[3] = Load<Vector256<int>>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<Vector256<int>>.Negate(Load<Vector256<int>>(ref values, inputStride, 1));
+            buffer0[5] = Load<Vector256<int>>(ref values, inputStride, 6);
+            buffer0[6] = Load<Vector256<int>>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<Vector256<int>>.Negate(Load<Vector256<int>>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector256<int>>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<int>>.CreateRounding(cosBit);
+
+            // Stage 1 applies the ADST permutation and signs while the source block is still read-only.
+            buffer0[0] = Load<Vector512<int>>(ref values, inputStride, 0);
+            buffer0[1] = Av1ForwardTransformArithmetic<Vector512<int>>.Negate(Load<Vector512<int>>(ref values, inputStride, 7));
+            buffer0[2] = Av1ForwardTransformArithmetic<Vector512<int>>.Negate(Load<Vector512<int>>(ref values, inputStride, 3));
+            buffer0[3] = Load<Vector512<int>>(ref values, inputStride, 4);
+            buffer0[4] = Av1ForwardTransformArithmetic<Vector512<int>>.Negate(Load<Vector512<int>>(ref values, inputStride, 1));
+            buffer0[5] = Load<Vector512<int>>(ref values, inputStride, 6);
+            buffer0[6] = Load<Vector512<int>>(ref values, inputStride, 2);
+            buffer0[7] = Av1ForwardTransformArithmetic<Vector512<int>>.Negate(Load<Vector512<int>>(ref values, inputStride, 5));
+
+            // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs.
+            buffer1[0] = buffer0[0];
+            buffer1[1] = buffer0[1];
+            Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding);
+            buffer1[4] = buffer0[4];
+            buffer1[5] = buffer0[5];
+            Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding);
+
+            // Stage 3 combines the rotated and copied pairs into two independent four-value groups.
+            for (int group = 0; group < 8; group += 4)
+            {
+                for (int i = 0; i < 2; i++)
+                {
+                    Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(
+                        buffer1[group + i],
+                        buffer1[group + i + 2],
+                        out buffer0[group + i],
+                        out buffer0[group + i + 2]);
+                }
+            }
+
+            // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged.
+            for (int i = 0; i < 4; i++)
+            {
+                buffer1[i] = buffer0[i];
+            }
+
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding);
+
+            // Stage 5 creates the four final butterfly pairs spanning the two groups.
+            for (int i = 0; i < 4; i++)
+            {
+                Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]);
+            }
+
+            // Stage 6 applies the remaining odd-angle rotations.
+            buffer1[0] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding);
+            buffer1[1] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding);
+            buffer1[2] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding);
+            buffer1[3] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding);
+            buffer1[4] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding);
+            buffer1[5] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding);
+            buffer1[6] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding);
+            buffer1[7] = Av1ForwardTransformArithmetic<Vector512<int>>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding);
+
+            ReadOnlySpan<byte> outputOrder = OutputOrder;
+
+            // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order.
+            for (int i = 0; i < 8; i++)
+            {
+                Store(ref values, outputStride, i, buffer1[outputOrder[i]]);
+            }
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct4Operator.cs

@@ -0,0 +1,395 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the four-point forward DCT operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the four-point forward discrete cosine transform.
+    /// </summary>
+    internal readonly struct Dct4Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<int>.CreateRounding(cosBit);
+
+            int input0 = Load<int>(ref values, inputStride, 0);
+            int input1 = Load<int>(ref values, inputStride, 1);
+            int input2 = Load<int>(ref values, inputStride, 2);
+            int input3 = Load<int>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<int>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<int>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out int output0,
+                out int output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out int output1,
+                out int output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<short>.CreateRounding(cosBit);
+
+            short input0 = Load<short>(ref values, inputStride, 0);
+            short input1 = Load<short>(ref values, inputStride, 1);
+            short input2 = Load<short>(ref values, inputStride, 2);
+            short input3 = Load<short>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<short>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<short>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out short output0,
+                out short output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out short output1,
+                out short output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<short>>.CreateRounding(cosBit);
+
+            Vector128<short> input0 = Load<Vector128<short>>(ref values, inputStride, 0);
+            Vector128<short> input1 = Load<Vector128<short>>(ref values, inputStride, 1);
+            Vector128<short> input2 = Load<Vector128<short>>(ref values, inputStride, 2);
+            Vector128<short> input3 = Load<Vector128<short>>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out Vector128<short> output0,
+                out Vector128<short> output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out Vector128<short> output1,
+                out Vector128<short> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<short>>.CreateRounding(cosBit);
+
+            Vector256<short> input0 = Load<Vector256<short>>(ref values, inputStride, 0);
+            Vector256<short> input1 = Load<Vector256<short>>(ref values, inputStride, 1);
+            Vector256<short> input2 = Load<Vector256<short>>(ref values, inputStride, 2);
+            Vector256<short> input3 = Load<Vector256<short>>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out Vector256<short> output0,
+                out Vector256<short> output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out Vector256<short> output1,
+                out Vector256<short> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<short>>.CreateRounding(cosBit);
+
+            Vector512<short> input0 = Load<Vector512<short>>(ref values, inputStride, 0);
+            Vector512<short> input1 = Load<Vector512<short>>(ref values, inputStride, 1);
+            Vector512<short> input2 = Load<Vector512<short>>(ref values, inputStride, 2);
+            Vector512<short> input3 = Load<Vector512<short>>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out Vector512<short> output0,
+                out Vector512<short> output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out Vector512<short> output1,
+                out Vector512<short> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<int>>.CreateRounding(cosBit);
+
+            Vector128<int> input0 = Load<Vector128<int>>(ref values, inputStride, 0);
+            Vector128<int> input1 = Load<Vector128<int>>(ref values, inputStride, 1);
+            Vector128<int> input2 = Load<Vector128<int>>(ref values, inputStride, 2);
+            Vector128<int> input3 = Load<Vector128<int>>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out Vector128<int> output0,
+                out Vector128<int> output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out Vector128<int> output1,
+                out Vector128<int> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<int>>.CreateRounding(cosBit);
+
+            Vector256<int> input0 = Load<Vector256<int>>(ref values, inputStride, 0);
+            Vector256<int> input1 = Load<Vector256<int>>(ref values, inputStride, 1);
+            Vector256<int> input2 = Load<Vector256<int>>(ref values, inputStride, 2);
+            Vector256<int> input3 = Load<Vector256<int>>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out Vector256<int> output0,
+                out Vector256<int> output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out Vector256<int> output1,
+                out Vector256<int> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<int>>.CreateRounding(cosBit);
+
+            Vector512<int> input0 = Load<Vector512<int>>(ref values, inputStride, 0);
+            Vector512<int> input1 = Load<Vector512<int>>(ref values, inputStride, 1);
+            Vector512<int> input2 = Load<Vector512<int>>(ref values, inputStride, 2);
+            Vector512<int> input3 = Load<Vector512<int>>(ref values, inputStride, 3);
+
+            // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
+            // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer0[0],
+                buffer0[1],
+                out Vector512<int> output0,
+                out Vector512<int> output2,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer0[3],
+                buffer0[2],
+                out Vector512<int> output1,
+                out Vector512<int> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct8Operator.cs

@@ -0,0 +1,859 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Implements the eight-point forward DCT stage network.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the eight-point forward transform for every supported lane width.
+    /// </summary>
+    internal readonly struct Dct8Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<int>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<int>.AddSubtract(
+                Load<int>(ref values, inputStride, 0),
+                Load<int>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<int>.AddSubtract(
+                Load<int>(ref values, inputStride, 1),
+                Load<int>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<int>.AddSubtract(
+                Load<int>(ref values, inputStride, 2),
+                Load<int>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<int>.AddSubtract(
+                Load<int>(ref values, inputStride, 3),
+                Load<int>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<int>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<int>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out int output0,
+                out int output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out int output2,
+                out int output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<int>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<int>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out int output1,
+                out int output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<int>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out int output5,
+                out int output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<short>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<short>.AddSubtract(
+                Load<short>(ref values, inputStride, 0),
+                Load<short>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<short>.AddSubtract(
+                Load<short>(ref values, inputStride, 1),
+                Load<short>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<short>.AddSubtract(
+                Load<short>(ref values, inputStride, 2),
+                Load<short>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<short>.AddSubtract(
+                Load<short>(ref values, inputStride, 3),
+                Load<short>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<short>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<short>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out short output0,
+                out short output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out short output2,
+                out short output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<short>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<short>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out short output1,
+                out short output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<short>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out short output5,
+                out short output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<short>>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(
+                Load<Vector128<short>>(ref values, inputStride, 0),
+                Load<Vector128<short>>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(
+                Load<Vector128<short>>(ref values, inputStride, 1),
+                Load<Vector128<short>>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(
+                Load<Vector128<short>>(ref values, inputStride, 2),
+                Load<Vector128<short>>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(
+                Load<Vector128<short>>(ref values, inputStride, 3),
+                Load<Vector128<short>>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out Vector128<short> output0,
+                out Vector128<short> output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out Vector128<short> output2,
+                out Vector128<short> output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<Vector128<short>>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out Vector128<short> output1,
+                out Vector128<short> output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<short>>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out Vector128<short> output5,
+                out Vector128<short> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<short>>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(
+                Load<Vector256<short>>(ref values, inputStride, 0),
+                Load<Vector256<short>>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(
+                Load<Vector256<short>>(ref values, inputStride, 1),
+                Load<Vector256<short>>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(
+                Load<Vector256<short>>(ref values, inputStride, 2),
+                Load<Vector256<short>>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(
+                Load<Vector256<short>>(ref values, inputStride, 3),
+                Load<Vector256<short>>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out Vector256<short> output0,
+                out Vector256<short> output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out Vector256<short> output2,
+                out Vector256<short> output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<Vector256<short>>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out Vector256<short> output1,
+                out Vector256<short> output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<short>>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out Vector256<short> output5,
+                out Vector256<short> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<short>>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(
+                Load<Vector512<short>>(ref values, inputStride, 0),
+                Load<Vector512<short>>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(
+                Load<Vector512<short>>(ref values, inputStride, 1),
+                Load<Vector512<short>>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(
+                Load<Vector512<short>>(ref values, inputStride, 2),
+                Load<Vector512<short>>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(
+                Load<Vector512<short>>(ref values, inputStride, 3),
+                Load<Vector512<short>>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out Vector512<short> output0,
+                out Vector512<short> output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out Vector512<short> output2,
+                out Vector512<short> output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<Vector512<short>>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out Vector512<short> output1,
+                out Vector512<short> output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<short>>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out Vector512<short> output5,
+                out Vector512<short> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector128<int>>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(
+                Load<Vector128<int>>(ref values, inputStride, 0),
+                Load<Vector128<int>>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(
+                Load<Vector128<int>>(ref values, inputStride, 1),
+                Load<Vector128<int>>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(
+                Load<Vector128<int>>(ref values, inputStride, 2),
+                Load<Vector128<int>>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(
+                Load<Vector128<int>>(ref values, inputStride, 3),
+                Load<Vector128<int>>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out Vector128<int> output0,
+                out Vector128<int> output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out Vector128<int> output2,
+                out Vector128<int> output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<Vector128<int>>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out Vector128<int> output1,
+                out Vector128<int> output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector128<int>>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out Vector128<int> output5,
+                out Vector128<int> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector256<int>>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(
+                Load<Vector256<int>>(ref values, inputStride, 0),
+                Load<Vector256<int>>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(
+                Load<Vector256<int>>(ref values, inputStride, 1),
+                Load<Vector256<int>>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(
+                Load<Vector256<int>>(ref values, inputStride, 2),
+                Load<Vector256<int>>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(
+                Load<Vector256<int>>(ref values, inputStride, 3),
+                Load<Vector256<int>>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out Vector256<int> output0,
+                out Vector256<int> output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out Vector256<int> output2,
+                out Vector256<int> output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<Vector256<int>>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out Vector256<int> output1,
+                out Vector256<int> output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector256<int>>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out Vector256<int> output5,
+                out Vector256<int> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            Av1TransformRounding rounding = Av1ForwardTransformArithmetic<Vector512<int>>.CreateRounding(cosBit);
+
+            // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer
+            // ownership, allowing the later even butterflies to write their final coefficients directly to the block.
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(
+                Load<Vector512<int>>(ref values, inputStride, 0),
+                Load<Vector512<int>>(ref values, inputStride, 7),
+                out buffer0[0],
+                out buffer1[7]);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(
+                Load<Vector512<int>>(ref values, inputStride, 1),
+                Load<Vector512<int>>(ref values, inputStride, 6),
+                out buffer0[1],
+                out buffer0[6]);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(
+                Load<Vector512<int>>(ref values, inputStride, 2),
+                Load<Vector512<int>>(ref values, inputStride, 5),
+                out buffer0[2],
+                out buffer0[5]);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(
+                Load<Vector512<int>>(ref values, inputStride, 3),
+                Load<Vector512<int>>(ref values, inputStride, 4),
+                out buffer0[3],
+                out buffer1[4]);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]);
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]);
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                -cospi[32],
+                cospi[32],
+                buffer0[5],
+                buffer0[6],
+                out buffer1[5],
+                out buffer1[6],
+                cosBit,
+                in rounding);
+
+            // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms.
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                cospi[32],
+                cospi[32],
+                buffer1[0],
+                buffer1[1],
+                out Vector512<int> output0,
+                out Vector512<int> output4,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                cospi[16],
+                cospi[48],
+                buffer1[3],
+                buffer1[2],
+                out Vector512<int> output2,
+                out Vector512<int> output6,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]);
+            Av1ForwardTransformArithmetic<Vector512<int>>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]);
+
+            // Highway fuses the final two stages because no intermediate value is reused after either rotation.
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                cospi[8],
+                cospi[56],
+                buffer0[7],
+                buffer0[4],
+                out Vector512<int> output1,
+                out Vector512<int> output7,
+                cosBit,
+                in rounding);
+
+            Av1ForwardTransformArithmetic<Vector512<int>>.Butterfly(
+                cospi[40],
+                cospi[24],
+                buffer0[6],
+                buffer0[5],
+                out Vector512<int> output5,
+                out Vector512<int> output3,
+                cosBit,
+                in rounding);
+
+            Store(ref values, outputStride, 0, output0);
+            Store(ref values, outputStride, 1, output1);
+            Store(ref values, outputStride, 2, output2);
+            Store(ref values, outputStride, 3, output3);
+            Store(ref values, outputStride, 4, output4);
+            Store(ref values, outputStride, 5, output5);
+            Store(ref values, outputStride, 6, output6);
+            Store(ref values, outputStride, 7, output7);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity16Operator.cs

@@ -0,0 +1,235 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the sixteen-point forward identity operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the sixteen-point forward identity transform.
+    /// </summary>
+    internal readonly struct Identity16Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                int input = Load<int>(ref values, inputStride, i);
+                int output = Av1ForwardTransformArithmetic<int>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                short input = Load<short>(ref values, inputStride, i);
+                short output = Av1ForwardTransformArithmetic<short>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                Vector128<short> input = Load<Vector128<short>>(ref values, inputStride, i);
+                Vector128<short> output = Av1ForwardTransformArithmetic<Vector128<short>>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                Vector256<short> input = Load<Vector256<short>>(ref values, inputStride, i);
+                Vector256<short> output = Av1ForwardTransformArithmetic<Vector256<short>>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                Vector512<short> input = Load<Vector512<short>>(ref values, inputStride, i);
+                Vector512<short> output = Av1ForwardTransformArithmetic<Vector512<short>>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                Vector128<int> input = Load<Vector128<int>>(ref values, inputStride, i);
+                Vector128<int> output = Av1ForwardTransformArithmetic<Vector128<int>>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                Vector256<int> input = Load<Vector256<int>>(ref values, inputStride, i);
+                Vector256<int> output = Av1ForwardTransformArithmetic<Vector256<int>>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 16; i++)
+            {
+                Vector512<int> input = Load<Vector512<int>>(ref values, inputStride, i);
+                Vector512<int> output = Av1ForwardTransformArithmetic<Vector512<int>>.MultiplyRound(
+                    input,
+                    2 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity32Operator.cs

@@ -0,0 +1,211 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the thirty-two-point forward identity operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the thirty-two-point forward identity transform.
+    /// </summary>
+    internal readonly struct Identity32Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                int input = Load<int>(ref values, inputStride, i);
+                int output = Av1ForwardTransformArithmetic<int>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                short input = Load<short>(ref values, inputStride, i);
+                short output = Av1ForwardTransformArithmetic<short>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                Vector128<short> input = Load<Vector128<short>>(ref values, inputStride, i);
+                Vector128<short> output = Av1ForwardTransformArithmetic<Vector128<short>>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                Vector256<short> input = Load<Vector256<short>>(ref values, inputStride, i);
+                Vector256<short> output = Av1ForwardTransformArithmetic<Vector256<short>>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                Vector512<short> input = Load<Vector512<short>>(ref values, inputStride, i);
+                Vector512<short> output = Av1ForwardTransformArithmetic<Vector512<short>>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                Vector128<int> input = Load<Vector128<int>>(ref values, inputStride, i);
+                Vector128<int> output = Av1ForwardTransformArithmetic<Vector128<int>>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                Vector256<int> input = Load<Vector256<int>>(ref values, inputStride, i);
+                Vector256<int> output = Av1ForwardTransformArithmetic<Vector256<int>>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 32; i++)
+            {
+                Vector512<int> input = Load<Vector512<int>>(ref values, inputStride, i);
+                Vector512<int> output = Av1ForwardTransformArithmetic<Vector512<int>>.ShiftLeft(input, 2);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity4Operator.cs

@@ -0,0 +1,235 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the four-point forward identity operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the four-point forward identity transform.
+    /// </summary>
+    internal readonly struct Identity4Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                int input = Load<int>(ref values, inputStride, i);
+                int output = Av1ForwardTransformArithmetic<int>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                short input = Load<short>(ref values, inputStride, i);
+                short output = Av1ForwardTransformArithmetic<short>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                Vector128<short> input = Load<Vector128<short>>(ref values, inputStride, i);
+                Vector128<short> output = Av1ForwardTransformArithmetic<Vector128<short>>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                Vector256<short> input = Load<Vector256<short>>(ref values, inputStride, i);
+                Vector256<short> output = Av1ForwardTransformArithmetic<Vector256<short>>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                Vector512<short> input = Load<Vector512<short>>(ref values, inputStride, i);
+                Vector512<short> output = Av1ForwardTransformArithmetic<Vector512<short>>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                Vector128<int> input = Load<Vector128<int>>(ref values, inputStride, i);
+                Vector128<int> output = Av1ForwardTransformArithmetic<Vector128<int>>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                Vector256<int> input = Load<Vector256<int>>(ref values, inputStride, i);
+                Vector256<int> output = Av1ForwardTransformArithmetic<Vector256<int>>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 4; i++)
+            {
+                Vector512<int> input = Load<Vector512<int>>(ref values, inputStride, i);
+                Vector512<int> output = Av1ForwardTransformArithmetic<Vector512<int>>.MultiplyRound(
+                    input,
+                    1 * Av1Transform1dMath.NewSqrt2,
+                    Av1Transform1dMath.NewSqrt2Bits);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity8Operator.cs

@@ -0,0 +1,211 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the eight-point forward identity operator.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Implements the eight-point forward identity transform.
+    /// </summary>
+    internal readonly struct Identity8Operator : IAv1ForwardTransform1dOperator
+    {
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                int input = Load<int>(ref values, inputStride, i);
+                int output = Av1ForwardTransformArithmetic<int>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                short input = Load<short>(ref values, inputStride, i);
+                short output = Av1ForwardTransformArithmetic<short>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                Vector128<short> input = Load<Vector128<short>>(ref values, inputStride, i);
+                Vector128<short> output = Av1ForwardTransformArithmetic<Vector128<short>>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                Vector256<short> input = Load<Vector256<short>>(ref values, inputStride, i);
+                Vector256<short> output = Av1ForwardTransformArithmetic<Vector256<short>>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                Vector512<short> input = Load<Vector512<short>>(ref values, inputStride, i);
+                Vector512<short> output = Av1ForwardTransformArithmetic<Vector512<short>>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                Vector128<int> input = Load<Vector128<int>>(ref values, inputStride, i);
+                Vector128<int> output = Av1ForwardTransformArithmetic<Vector128<int>>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                Vector256<int> input = Load<Vector256<int>>(ref values, inputStride, i);
+                Vector256<int> output = Av1ForwardTransformArithmetic<Vector256<int>>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit)
+        {
+            _ = buffer0;
+            _ = buffer1;
+            _ = cosBit;
+
+            // The length-specific normalization is applied directly in the semantic operator so each scalar
+            // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer.
+            for (int i = 0; i < 8; i++)
+            {
+                Vector512<int> input = Load<Vector512<int>>(ref values, inputStride, i);
+                Vector512<int> output = Av1ForwardTransformArithmetic<Vector512<int>>.ShiftLeft(input, 1);
+
+                Store(ref values, outputStride, i, output);
+            }
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs → src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operations.cs

@@ -2,71 +2,15 @@
 // Licensed under the Six Labors Split License.
 
 using System.Runtime.CompilerServices;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
 
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
 
 /// <content>
-/// Implements the four-point forward DCT stage network.
+/// Defines shared forward-transform storage and rotation primitives.
 /// </content>
-internal static partial class Av1ForwardTransformOperations
+internal static partial class Av1ForwardTransformer
 {
-    /// <summary>
-    /// Applies the four-point forward discrete cosine transform to every independent lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static void Dct4<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        Av1TransformRounding rounding = Av1ForwardTransformArithmetic<TValue>.CreateRounding(cosBit);
-
-        TValue input0 = Load<TValue>(ref values, inputStride, 0);
-        TValue input1 = Load<TValue>(ref values, inputStride, 1);
-        TValue input2 = Load<TValue>(ref values, inputStride, 2);
-        TValue input3 = Load<TValue>(ref values, inputStride, 3);
-
-        // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain
-        // Highway's saturating add/subtract behavior before the widening butterfly multiplication.
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]);
-        Av1ForwardTransformArithmetic<TValue>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]);
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[32],
-            cospi[32],
-            buffer0[0],
-            buffer0[1],
-            out TValue output0,
-            out TValue output2,
-            cosBit,
-            in rounding);
-
-        Av1ForwardTransformArithmetic<TValue>.Butterfly(
-            cospi[16],
-            cospi[48],
-            buffer0[3],
-            buffer0[2],
-            out TValue output1,
-            out TValue output3,
-            cosBit,
-            in rounding);
-
-        Store(ref values, outputStride, 0, output0);
-        Store(ref values, outputStride, 1, output1);
-        Store(ref values, outputStride, 2, output2);
-        Store(ref values, outputStride, 3, output3);
-    }
-
     /// <summary>
     /// Loads one scalar or SIMD transform value from strided block storage.
     /// </summary>

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operator.cs

@@ -0,0 +1,158 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the operator contract for one-dimensional AV1 forward transforms.
+/// </content>
+internal static partial class Av1ForwardTransformer
+{
+    /// <summary>
+    /// Defines the scalar and SIMD arithmetic for one AV1 forward transform.
+    /// </summary>
+    /// <remarks>
+    /// Every overload applies the same stage network to independent transform axes. The family traversal selects one
+    /// concrete lane width, while the closed semantic operator lets the JIT resolve the static call before the stages.
+    /// </remarks>
+    internal interface IAv1ForwardTransform1dOperator
+    {
+        /// <summary>
+        /// Transforms one expanded axis without hardware vectorization.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<int> buffer0,
+            ref Av1TransformVector<int> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms one packed axis without hardware vectorization.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<short> buffer0,
+            ref Av1TransformVector<short> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms eight packed axes in parallel.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<short>> buffer0,
+            ref Av1TransformVector<Vector128<short>> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms sixteen packed axes in parallel.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<short>> buffer0,
+            ref Av1TransformVector<Vector256<short>> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms thirty-two packed axes in parallel.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<short>> buffer0,
+            ref Av1TransformVector<Vector512<short>> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms four expanded axes in parallel.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector128<int>> buffer0,
+            ref Av1TransformVector<Vector128<int>> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms eight expanded axes in parallel.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector256<int>> buffer0,
+            ref Av1TransformVector<Vector256<int>> buffer1,
+            int cosBit);
+
+        /// <summary>
+        /// Transforms sixteen expanded axes in parallel.
+        /// </summary>
+        /// <param name="values">The first value in the strided transform storage.</param>
+        /// <param name="inputStride">The byte distance between consecutive input positions.</param>
+        /// <param name="outputStride">The byte distance between consecutive output positions.</param>
+        /// <param name="buffer0">The first transform-stage workspace buffer.</param>
+        /// <param name="buffer1">The second transform-stage workspace buffer.</param>
+        /// <param name="cosBit">The fixed-point precision of the transform constants.</param>
+        public static abstract void Transform(
+            ref byte values,
+            nint inputStride,
+            nint outputStride,
+            ref Av1TransformVector<Vector512<int>> buffer0,
+            ref Av1TransformVector<Vector512<int>> buffer1,
+            int cosBit);
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the sixteen-point AV1 forward identity transform operator.
-/// </summary>
-internal readonly struct Av1Identity16Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the thirty-two-point AV1 forward identity transform operator.
-/// </summary>
-internal readonly struct Av1Identity32Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the four-point AV1 forward identity transform operator.
-/// </summary>
-internal readonly struct Av1Identity4Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs

@@ -1,21 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines the eight-point AV1 forward identity transform operator.
-/// </summary>
-internal readonly struct Av1Identity8Forward1dOperator : IAv1ForwardTransform1dOperator
-{
-    /// <inheritdoc/>
-    public static void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct
-        => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs

@@ -1,34 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
-
-/// <summary>
-/// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type.
-/// </summary>
-/// <remarks>
-/// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type
-/// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch
-/// inside the transform stages.
-/// </remarks>
-internal interface IAv1ForwardTransform1dOperator
-{
-    /// <summary>
-    /// Transforms the independent axes stored in each value lane.
-    /// </summary>
-    /// <typeparam name="TValue">The scalar or SIMD value containing the independent transform axes.</typeparam>
-    /// <param name="values">The first value in the strided transform block.</param>
-    /// <param name="inputStride">The byte distance between consecutive input positions.</param>
-    /// <param name="outputStride">The byte distance between consecutive output positions.</param>
-    /// <param name="buffer0">The first fixed transform-stage buffer.</param>
-    /// <param name="buffer1">The second fixed transform-stage buffer.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    public static abstract void Transform<TValue>(
-        ref byte values,
-        nint inputStride,
-        nint outputStride,
-        ref Av1TransformVector<TValue> buffer0,
-        ref Av1TransformVector<TValue> buffer1,
-        int cosBit)
-        where TValue : struct;
-}

src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs

@@ -1,46 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
-
-/// <summary>
-/// Defines how inverse-transform residuals are added to a decoded sample representation.
-/// </summary>
-/// <remarks>
-/// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions,
-/// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable
-/// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize.
-/// </remarks>
-/// <typeparam name="TSample">The decoded sample storage type.</typeparam>
-internal interface IAv1InverseTransformOutputOperator<TSample>
-    where TSample : unmanaged
-{
-    /// <summary>
-    /// Adds one residual to a predicted sample and clips the result to the coded bit depth.
-    /// </summary>
-    /// <param name="prediction">The predicted sample.</param>
-    /// <param name="residual">The inverse-transform residual.</param>
-    /// <param name="bitDepth">The coded sample bit depth.</param>
-    /// <returns>The reconstructed sample.</returns>
-    public static abstract TSample Add(TSample prediction, int residual, int bitDepth);
-
-    /// <summary>
-    /// Adds four residuals to four predicted samples and stores the clipped results.
-    /// </summary>
-    /// <param name="prediction">The first predicted sample.</param>
-    /// <param name="destination">The first destination sample.</param>
-    /// <param name="residual">The four inverse-transform residuals.</param>
-    /// <param name="bitDepth">The coded sample bit depth.</param>
-    public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128<int> residual, int bitDepth);
-
-    /// <summary>
-    /// Adds eight residuals to eight predicted samples and stores the clipped results.
-    /// </summary>
-    /// <param name="prediction">The first predicted sample.</param>
-    /// <param name="destination">The first destination sample.</param>
-    /// <param name="residual">The eight inverse-transform residuals.</param>
-    /// <param name="bitDepth">The coded sample bit depth.</param>
-    public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256<int> residual, int bitDepth);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs

@@ -1,59 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
-
-/// <summary>
-/// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional transform.
-/// </summary>
-/// <remarks>
-/// Each overload performs the same staged fixed-point transform. In the SIMD overloads, each vector field identifies
-/// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore
-/// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage
-/// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the
-/// JIT to specialize every static interface call outside the stage network.
-/// </remarks>
-internal interface IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Transforms one axis when hardware vectorization is unavailable.
-    /// </summary>
-    /// <param name="input">The source values for the transform axis.</param>
-    /// <param name="output">The destination values for the transform axis.</param>
-    /// <param name="step">The fixed stage storage for the transform axis.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static abstract void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange);
-
-    /// <summary>
-    /// Transforms four independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for four transform axes.</param>
-    /// <param name="output">The destination values for four transform axes.</param>
-    /// <param name="step">The fixed stage storage for four transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static abstract void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange);
-
-    /// <summary>
-    /// Transforms eight independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for eight transform axes.</param>
-    /// <param name="output">The destination values for eight transform axes.</param>
-    /// <param name="step">The fixed stage storage for eight transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static abstract void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange);
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs

@@ -1,568 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
-/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
-/// </remarks>
-internal readonly struct Av1Adst16Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform.
-    /// </summary>
-    /// <param name="input">The sixteen frequency-domain coefficients.</param>
-    /// <param name="output">The sixteen spatial-domain residual values.</param>
-    /// <param name="step">The sixteen-element stage buffer owned by the containing two-dimensional transform.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
-        stage++;
-        output[0] = input[15];
-        output[1] = input[0];
-        output[2] = input[13];
-        output[3] = input[2];
-        output[4] = input[11];
-        output[5] = input[4];
-        output[6] = input[9];
-        output[7] = input[6];
-        output[8] = input[7];
-        output[9] = input[8];
-        output[10] = input[5];
-        output[11] = input[10];
-        output[12] = input[3];
-        output[13] = input[12];
-        output[14] = input[1];
-        output[15] = input[14];
-
-        // Stage 2 applies the terminal odd-angle rotations in reverse.
-        stage++;
-        step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit);
-        step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit);
-        step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit);
-        step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit);
-        step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit);
-        step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit);
-        step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit);
-        step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit);
-        step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit);
-        step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit);
-        step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit);
-        step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit);
-        step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit);
-        step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit);
-        step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit);
-
-        // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane.
-        stage++;
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]);
-        output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]);
-        output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]);
-        output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]);
-        output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]);
-        output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]);
-        output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]);
-        output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]);
-        output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]);
-        output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]);
-        output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]);
-        output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]);
-        output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]);
-        output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]);
-        output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]);
-
-        // Stage 4 reverses the pi/16 rotations in the upper half.
-        stage++;
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = output[2];
-        step[3] = output[3];
-        step[4] = output[4];
-        step[5] = output[5];
-        step[6] = output[6];
-        step[7] = output[7];
-        step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit);
-        step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit);
-        step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit);
-        step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit);
-        step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit);
-        step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit);
-        step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit);
-        step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit);
-
-        // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane.
-        stage++;
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]);
-        output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]);
-        output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]);
-        output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]);
-        output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]);
-        output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]);
-        output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]);
-        output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]);
-        output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]);
-        output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]);
-        output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]);
-        output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]);
-        output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]);
-        output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]);
-        output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]);
-
-        // Stage 6 reverses the pi/8 and 3pi/8 rotations.
-        stage++;
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = output[2];
-        step[3] = output[3];
-        step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit);
-        step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit);
-        step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit);
-        step[8] = output[8];
-        step[9] = output[9];
-        step[10] = output[10];
-        step[11] = output[11];
-        step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit);
-        step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit);
-        step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit);
-        step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit);
-
-        // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane.
-        stage++;
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]);
-        output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]);
-        output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]);
-        output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]);
-        output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]);
-        output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]);
-        output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]);
-        output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]);
-        output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]);
-        output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]);
-        output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]);
-        output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]);
-        output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]);
-        output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]);
-        output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]);
-
-        // Stage 8 reverses the pi/4 rotations for the middle pairs.
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit);
-        step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit);
-        step[4] = output[4];
-        step[5] = output[5];
-        step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit);
-        step[8] = output[8];
-        step[9] = output[9];
-        step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit);
-        step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit);
-        step[12] = output[12];
-        step[13] = output[13];
-        step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit);
-        step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit);
-
-        // Stage 9 applies the AV1 signs and permutation that restore spatial sample order.
-        output[0] = step[0];
-        output[1] = -step[8];
-        output[2] = step[12];
-        output[3] = -step[4];
-        output[4] = step[6];
-        output[5] = -step[14];
-        output[6] = step[10];
-        output[7] = -step[2];
-        output[8] = step[3];
-        output[9] = -step[11];
-        output[10] = step[15];
-        output[11] = -step[7];
-        output[12] = step[5];
-        output[13] = -step[13];
-        output[14] = step[9];
-        output[15] = -step[1];
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
-        stage++;
-        output.V0 = input.V15;
-        output.V1 = input.V0;
-        output.V2 = input.V13;
-        output.V3 = input.V2;
-        output.V4 = input.V11;
-        output.V5 = input.V4;
-        output.V6 = input.V9;
-        output.V7 = input.V6;
-        output.V8 = input.V7;
-        output.V9 = input.V8;
-        output.V10 = input.V5;
-        output.V11 = input.V10;
-        output.V12 = input.V3;
-        output.V13 = input.V12;
-        output.V14 = input.V1;
-        output.V15 = input.V14;
-
-        // Stage 2 applies the terminal odd-angle rotations in reverse.
-        stage++;
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit);
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit);
-        step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit);
-        step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit);
-
-        // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]);
-        output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]);
-        output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]);
-        output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]);
-        output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]);
-        output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]);
-        output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]);
-        output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]);
-        output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]);
-
-        // Stage 4 reverses the pi/16 rotations in the upper half.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = output.V6;
-        step.V7 = output.V7;
-        step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit);
-        step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit);
-
-        // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]);
-        output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]);
-        output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]);
-        output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]);
-        output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]);
-        output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]);
-        output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]);
-        output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]);
-
-        // Stage 6 reverses the pi/8 and 3pi/8 rotations.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
-        step.V8 = output.V8;
-        step.V9 = output.V9;
-        step.V10 = output.V10;
-        step.V11 = output.V11;
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit);
-
-        // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]);
-        output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]);
-        output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]);
-        output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]);
-        output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]);
-        output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]);
-        output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]);
-        output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]);
-
-        // Stage 8 reverses the pi/4 rotations for the middle pairs.
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
-        step.V8 = output.V8;
-        step.V9 = output.V9;
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit);
-        step.V12 = output.V12;
-        step.V13 = output.V13;
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit);
-
-        // Stage 9 applies the AV1 signs and permutation that restore spatial sample order.
-        output.V0 = step.V0;
-        output.V1 = -step.V8;
-        output.V2 = step.V12;
-        output.V3 = -step.V4;
-        output.V4 = step.V6;
-        output.V5 = -step.V14;
-        output.V6 = step.V10;
-        output.V7 = -step.V2;
-        output.V8 = step.V3;
-        output.V9 = -step.V11;
-        output.V10 = step.V15;
-        output.V11 = -step.V7;
-        output.V12 = step.V5;
-        output.V13 = -step.V13;
-        output.V14 = step.V9;
-        output.V15 = -step.V1;
-    }
-
-    /// <summary>
-    /// Applies the transform to four independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for the parallel transform axes.</param>
-    /// <param name="output">The destination values for the parallel transform axes.</param>
-    /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
-        stage++;
-        output.V0 = input.V15;
-        output.V1 = input.V0;
-        output.V2 = input.V13;
-        output.V3 = input.V2;
-        output.V4 = input.V11;
-        output.V5 = input.V4;
-        output.V6 = input.V9;
-        output.V7 = input.V6;
-        output.V8 = input.V7;
-        output.V9 = input.V8;
-        output.V10 = input.V5;
-        output.V11 = input.V10;
-        output.V12 = input.V3;
-        output.V13 = input.V12;
-        output.V14 = input.V1;
-        output.V15 = input.V14;
-
-        // Stage 2 applies the terminal odd-angle rotations in reverse.
-        stage++;
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit);
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit);
-        step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit);
-        step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit);
-
-        // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]);
-        output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]);
-        output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]);
-        output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]);
-        output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]);
-        output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]);
-        output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]);
-        output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]);
-        output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]);
-
-        // Stage 4 reverses the pi/16 rotations in the upper half.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = output.V6;
-        step.V7 = output.V7;
-        step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit);
-        step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit);
-
-        // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]);
-        output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]);
-        output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]);
-        output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]);
-        output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]);
-        output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]);
-        output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]);
-        output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]);
-
-        // Stage 6 reverses the pi/8 and 3pi/8 rotations.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
-        step.V8 = output.V8;
-        step.V9 = output.V9;
-        step.V10 = output.V10;
-        step.V11 = output.V11;
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit);
-
-        // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]);
-        output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]);
-        output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]);
-        output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]);
-        output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]);
-        output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]);
-        output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]);
-        output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]);
-
-        // Stage 8 reverses the pi/4 rotations for the middle pairs.
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
-        step.V8 = output.V8;
-        step.V9 = output.V9;
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit);
-        step.V12 = output.V12;
-        step.V13 = output.V13;
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit);
-
-        // Stage 9 applies the AV1 signs and permutation that restore spatial sample order.
-        output.V0 = step.V0;
-        output.V1 = -step.V8;
-        output.V2 = step.V12;
-        output.V3 = -step.V4;
-        output.V4 = step.V6;
-        output.V5 = -step.V14;
-        output.V6 = step.V10;
-        output.V7 = -step.V2;
-        output.V8 = step.V3;
-        output.V9 = -step.V11;
-        output.V10 = step.V15;
-        output.V11 = -step.V7;
-        output.V12 = step.V5;
-        output.V13 = -step.V13;
-        output.V14 = step.V9;
-        output.V15 = -step.V1;
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs

@@ -1,168 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
-/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
-/// </remarks>
-internal readonly struct Av1Adst4Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform.
-    /// </summary>
-    /// <param name="input">The four frequency-domain coefficients.</param>
-    /// <param name="output">The four spatial-domain residual values.</param>
-    /// <param name="step">The stage buffer owned by the containing two-dimensional transform.</param>
-    /// <param name="cosBit">The fixed-point precision of the sine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
-
-        // libaom widens the complete four-point factorization because the products retain their fixed-point scale
-        // until the final shift. The stage buffer is therefore unnecessary for this transform size.
-        long x0 = input[0];
-        long x1 = input[1];
-        long x2 = input[2];
-        long x3 = input[3];
-
-        _ = step;
-        _ = stageRange;
-
-        // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel.
-        if ((x0 | x1 | x2 | x3) == 0)
-        {
-            output[..4].Clear();
-            return;
-        }
-
-        // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3.
-        long s0 = sinpi[1] * x0;
-        long s1 = sinpi[2] * x0;
-        long s2 = sinpi[3] * x1;
-        long s3 = sinpi[4] * x2;
-        long s4 = sinpi[1] * x2;
-        long s5 = sinpi[2] * x3;
-        long s6 = sinpi[4] * x3;
-        long s7 = (x0 - x2) + x3;
-
-        // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding.
-        s0 += s3;
-        s1 -= s4;
-        s3 = s2;
-        s2 = sinpi[3] * s7;
-        s0 += s5;
-        s1 -= s6;
-        x0 = s0 + s3;
-        x1 = s1 + s3;
-        x2 = s2;
-        x3 = (s0 + s1) - s3;
-
-        output[0] = Av1Math.RoundShift(x0, cosBit);
-        output[1] = Av1Math.RoundShift(x1, cosBit);
-        output[2] = Av1Math.RoundShift(x2, cosBit);
-        output[3] = Av1Math.RoundShift(x3, cosBit);
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        TransformCore(ref input, ref output, cosBit, stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount);
-        _ = step;
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        TransformCore(ref input, ref output, cosBit, stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount);
-        _ = step;
-    }
-
-    /// <summary>
-    /// Applies the inverse four-point matrix to four independent axes.
-    /// </summary>
-    /// <param name="input">The source values for four transform axes.</param>
-    /// <param name="output">The destination values for four transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the sine constants.</param>
-    /// <param name="widenedRound">Whether the terminal fixed-point rounding requires signed 64-bit lanes.</param>
-    private static void TransformCore(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        int cosBit,
-        bool widenedRound)
-    {
-        ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
-        Vector128<int> x0 = input.V0;
-        Vector128<int> x1 = input.V1;
-        Vector128<int> x2 = input.V2;
-        Vector128<int> x3 = input.V3;
-
-        // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row
-        // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow
-        // a valid Int32 fixed-point sum.
-        if (widenedRound)
-        {
-            output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
-            output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
-            output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
-            output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
-            return;
-        }
-
-        output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
-        output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
-        output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
-        output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
-    }
-
-    /// <summary>
-    /// Applies the inverse four-point matrix to eight independent axes.
-    /// </summary>
-    /// <param name="input">The source values for eight transform axes.</param>
-    /// <param name="output">The destination values for eight transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the sine constants.</param>
-    /// <param name="widenedRound">Whether the terminal fixed-point rounding requires signed 64-bit lanes.</param>
-    private static void TransformCore(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        int cosBit,
-        bool widenedRound)
-    {
-        ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
-        Vector256<int> x0 = input.V0;
-        Vector256<int> x1 = input.V1;
-        Vector256<int> x2 = input.V2;
-        Vector256<int> x3 = input.V3;
-
-        if (widenedRound)
-        {
-            output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
-            output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
-            output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
-            output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
-            return;
-        }
-
-        output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
-        output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
-        output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
-        output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs

@@ -1,289 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
-/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
-/// </remarks>
-internal readonly struct Av1Adst8Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform.
-    /// </summary>
-    /// <param name="input">The eight frequency-domain coefficients.</param>
-    /// <param name="output">The eight spatial-domain residual values.</param>
-    /// <param name="step">The eight-element stage buffer owned by the containing two-dimensional transform.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
-        stage++;
-        output[0] = input[7];
-        output[1] = input[0];
-        output[2] = input[5];
-        output[3] = input[2];
-        output[4] = input[3];
-        output[5] = input[4];
-        output[6] = input[1];
-        output[7] = input[6];
-
-        // Stage 2 applies the terminal odd-angle rotations in reverse.
-        stage++;
-        step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit);
-        step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit);
-        step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit);
-        step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit);
-        step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit);
-        step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit);
-        step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit);
-
-        // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane.
-        stage++;
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]);
-        output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]);
-        output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]);
-        output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]);
-        output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]);
-        output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]);
-        output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]);
-
-        // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half.
-        stage++;
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = output[2];
-        step[3] = output[3];
-        step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit);
-        step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit);
-        step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit);
-
-        // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane.
-        stage++;
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]);
-        output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]);
-        output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]);
-        output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]);
-        output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]);
-        output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]);
-        output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]);
-
-        // Stage 6 reverses the pi/4 rotations for the middle pairs.
-        stage++;
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit);
-        step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit);
-        step[4] = output[4];
-        step[5] = output[5];
-        step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit);
-
-        // Stage 7 applies the AV1 signs and permutation that restore spatial sample order.
-        output[0] = step[0];
-        output[1] = -step[4];
-        output[2] = step[6];
-        output[3] = -step[2];
-        output[4] = step[3];
-        output[5] = -step[7];
-        output[6] = step[5];
-        output[7] = -step[1];
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
-        stage++;
-        output.V0 = input.V7;
-        output.V1 = input.V0;
-        output.V2 = input.V5;
-        output.V3 = input.V2;
-        output.V4 = input.V3;
-        output.V5 = input.V4;
-        output.V6 = input.V1;
-        output.V7 = input.V6;
-
-        // Stage 2 applies the terminal odd-angle rotations in reverse.
-        stage++;
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit);
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit);
-
-        // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
-
-        // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
-
-        // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
-
-        // Stage 6 reverses the pi/4 rotations for the middle pairs.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
-
-        // Stage 7 applies the AV1 signs and permutation that restore spatial sample order.
-        output.V0 = step.V0;
-        output.V1 = -step.V4;
-        output.V2 = step.V6;
-        output.V3 = -step.V2;
-        output.V4 = step.V3;
-        output.V5 = -step.V7;
-        output.V6 = step.V5;
-        output.V7 = -step.V1;
-    }
-
-    /// <summary>
-    /// Applies the transform to four independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for the parallel transform axes.</param>
-    /// <param name="output">The destination values for the parallel transform axes.</param>
-    /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
-        stage++;
-        output.V0 = input.V7;
-        output.V1 = input.V0;
-        output.V2 = input.V5;
-        output.V3 = input.V2;
-        output.V4 = input.V3;
-        output.V5 = input.V4;
-        output.V6 = input.V1;
-        output.V7 = input.V6;
-
-        // Stage 2 applies the terminal odd-angle rotations in reverse.
-        stage++;
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit);
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit);
-
-        // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
-
-        // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
-
-        // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane.
-        stage++;
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
-        output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
-        output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
-        output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
-        output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
-
-        // Stage 6 reverses the pi/4 rotations for the middle pairs.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
-
-        // Stage 7 applies the AV1 signs and permutation that restore spatial sample order.
-        output.V0 = step.V0;
-        output.V1 = -step.V4;
-        output.V2 = step.V6;
-        output.V3 = -step.V2;
-        output.V4 = step.V3;
-        output.V5 = -step.V7;
-        output.V6 = step.V5;
-        output.V7 = -step.V1;
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs

@@ -1,475 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the 16-point AV1 inverse discrete cosine transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
-/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
-/// </remarks>
-internal readonly struct Av1Dct16Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative 16-point AV1 inverse discrete cosine transform.
-    /// </summary>
-    /// <param name="input">The sixteen frequency-domain coefficients.</param>
-    /// <param name="output">The sixteen spatial-domain residual values.</param>
-    /// <param name="step">The sixteen-element stage buffer owned by the containing two-dimensional transform.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
-        stage++;
-        output[0] = input[0];
-        output[1] = input[8];
-        output[2] = input[4];
-        output[3] = input[12];
-        output[4] = input[2];
-        output[5] = input[10];
-        output[6] = input[6];
-        output[7] = input[14];
-        output[8] = input[1];
-        output[9] = input[9];
-        output[10] = input[5];
-        output[11] = input[13];
-        output[12] = input[3];
-        output[13] = input[11];
-        output[14] = input[7];
-        output[15] = input[15];
-
-        // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles.
-        stage++;
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = output[2];
-        step[3] = output[3];
-        step[4] = output[4];
-        step[5] = output[5];
-        step[6] = output[6];
-        step[7] = output[7];
-        step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit);
-        step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit);
-        step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit);
-        step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit);
-        step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit);
-        step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit);
-        step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit);
-        step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit);
-
-        // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms.
-        stage++;
-        byte range = stageRange[stage];
-        output[0] = step[0];
-        output[1] = step[1];
-        output[2] = step[2];
-        output[3] = step[3];
-        output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit);
-        output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit);
-        output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit);
-        output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit);
-        output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range);
-        output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range);
-        output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range);
-        output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range);
-        output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range);
-        output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range);
-        output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range);
-        output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range);
-
-        // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs.
-        stage++;
-        range = stageRange[stage];
-        step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit);
-        step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit);
-        step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit);
-        step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit);
-        step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range);
-        step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range);
-        step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range);
-        step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range);
-        step[8] = output[8];
-        step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit);
-        step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit);
-        step[11] = output[11];
-        step[12] = output[12];
-        step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit);
-        step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit);
-        step[15] = output[15];
-
-        // Stage 5 widens the reconstructed groups through their next butterfly level.
-        stage++;
-        range = stageRange[stage];
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range);
-        output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range);
-        output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range);
-        output[4] = step[4];
-        output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit);
-        output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit);
-        output[7] = step[7];
-        output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range);
-        output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range);
-        output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range);
-        output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range);
-        output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range);
-        output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range);
-        output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range);
-        output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range);
-
-        // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge.
-        stage++;
-        range = stageRange[stage];
-        step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range);
-        step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range);
-        step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range);
-        step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range);
-        step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range);
-        step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range);
-        step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range);
-        step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range);
-        step[8] = output[8];
-        step[9] = output[9];
-        step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit);
-        step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit);
-        step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit);
-        step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit);
-        step[14] = output[14];
-        step[15] = output[15];
-
-        // Stage 7 merges the even and odd halves into spatial order and clamps every result.
-        stage++;
-        range = stageRange[stage];
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range);
-        output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range);
-        output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range);
-        output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range);
-        output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range);
-        output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range);
-        output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range);
-        output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range);
-        output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range);
-        output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range);
-        output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range);
-        output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range);
-        output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range);
-        output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range);
-        output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range);
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
-        stage++;
-        output.V0 = input.V0;
-        output.V1 = input.V8;
-        output.V2 = input.V4;
-        output.V3 = input.V12;
-        output.V4 = input.V2;
-        output.V5 = input.V10;
-        output.V6 = input.V6;
-        output.V7 = input.V14;
-        output.V8 = input.V1;
-        output.V9 = input.V9;
-        output.V10 = input.V5;
-        output.V11 = input.V13;
-        output.V12 = input.V3;
-        output.V13 = input.V11;
-        output.V14 = input.V7;
-        output.V15 = input.V15;
-
-        // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = output.V6;
-        step.V7 = output.V7;
-        step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit);
-        step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit);
-
-        // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms.
-        stage++;
-        byte range = stageRange[stage];
-        output.V0 = step.V0;
-        output.V1 = step.V1;
-        output.V2 = step.V2;
-        output.V3 = step.V3;
-        output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit);
-        output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit);
-        output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit);
-        output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit);
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range);
-        output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range);
-        output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range);
-        output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range);
-        output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range);
-        output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range);
-        output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range);
-        output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range);
-
-        // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs.
-        stage++;
-        range = stageRange[stage];
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
-        step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range);
-        step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range);
-        step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range);
-        step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range);
-        step.V8 = output.V8;
-        step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit);
-        step.V11 = output.V11;
-        step.V12 = output.V12;
-        step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit);
-        step.V15 = output.V15;
-
-        // Stage 5 widens the reconstructed groups through their next butterfly level.
-        stage++;
-        range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
-        output.V4 = step.V4;
-        output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit);
-        output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit);
-        output.V7 = step.V7;
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range);
-        output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range);
-        output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range);
-        output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range);
-        output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range);
-        output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range);
-        output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range);
-        output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range);
-
-        // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge.
-        stage++;
-        range = stageRange[stage];
-        step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range);
-        step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range);
-        step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range);
-        step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range);
-        step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range);
-        step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range);
-        step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range);
-        step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range);
-        step.V8 = output.V8;
-        step.V9 = output.V9;
-        step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit);
-        step.V14 = output.V14;
-        step.V15 = output.V15;
-
-        // Stage 7 merges the even and odd halves into spatial order and clamps every result.
-        stage++;
-        range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range);
-        output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range);
-        output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range);
-        output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range);
-        output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range);
-        output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range);
-        output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range);
-        output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range);
-        output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range);
-        output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range);
-        output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range);
-    }
-
-    /// <summary>
-    /// Applies the transform to four independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for the parallel transform axes.</param>
-    /// <param name="output">The destination values for the parallel transform axes.</param>
-    /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
-        stage++;
-        output.V0 = input.V0;
-        output.V1 = input.V8;
-        output.V2 = input.V4;
-        output.V3 = input.V12;
-        output.V4 = input.V2;
-        output.V5 = input.V10;
-        output.V6 = input.V6;
-        output.V7 = input.V14;
-        output.V8 = input.V1;
-        output.V9 = input.V9;
-        output.V10 = input.V5;
-        output.V11 = input.V13;
-        output.V12 = input.V3;
-        output.V13 = input.V11;
-        output.V14 = input.V7;
-        output.V15 = input.V15;
-
-        // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = output.V4;
-        step.V5 = output.V5;
-        step.V6 = output.V6;
-        step.V7 = output.V7;
-        step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit);
-        step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit);
-        step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit);
-
-        // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms.
-        stage++;
-        byte range = stageRange[stage];
-        output.V0 = step.V0;
-        output.V1 = step.V1;
-        output.V2 = step.V2;
-        output.V3 = step.V3;
-        output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit);
-        output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit);
-        output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit);
-        output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit);
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range);
-        output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range);
-        output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range);
-        output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range);
-        output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range);
-        output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range);
-        output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range);
-        output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range);
-
-        // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs.
-        stage++;
-        range = stageRange[stage];
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
-        step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range);
-        step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range);
-        step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range);
-        step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range);
-        step.V8 = output.V8;
-        step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit);
-        step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit);
-        step.V11 = output.V11;
-        step.V12 = output.V12;
-        step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit);
-        step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit);
-        step.V15 = output.V15;
-
-        // Stage 5 widens the reconstructed groups through their next butterfly level.
-        stage++;
-        range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
-        output.V4 = step.V4;
-        output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit);
-        output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit);
-        output.V7 = step.V7;
-        output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range);
-        output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range);
-        output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range);
-        output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range);
-        output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range);
-        output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range);
-        output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range);
-        output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range);
-
-        // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge.
-        stage++;
-        range = stageRange[stage];
-        step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range);
-        step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range);
-        step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range);
-        step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range);
-        step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range);
-        step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range);
-        step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range);
-        step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range);
-        step.V8 = output.V8;
-        step.V9 = output.V9;
-        step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit);
-        step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit);
-        step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit);
-        step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit);
-        step.V14 = output.V14;
-        step.V15 = output.V15;
-
-        // Stage 7 merges the even and odd halves into spatial order and clamps every result.
-        stage++;
-        range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range);
-        output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range);
-        output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range);
-        output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range);
-        output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range);
-        output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range);
-        output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range);
-        output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range);
-        output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range);
-        output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range);
-        output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range);
-        output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs

@@ -1,112 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the four-point AV1 inverse discrete cosine transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
-/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
-/// </remarks>
-internal readonly struct Av1Dct4Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative four-point AV1 inverse discrete cosine transform.
-    /// </summary>
-    /// <param name="input">The four frequency-domain coefficients.</param>
-    /// <param name="output">The four spatial-domain residual values.</param>
-    /// <param name="step">The four-element stage buffer owned by the containing two-dimensional transform.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT.
-        output[0] = input[0];
-        output[1] = input[2];
-        output[2] = input[1];
-        output[3] = input[3];
-
-        // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform.
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit);
-        step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit);
-        step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit);
-        step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit);
-
-        // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range.
-        byte range = stageRange[3];
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range);
-        output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range);
-        output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range);
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT.
-        output.V0 = input.V0;
-        output.V1 = input.V2;
-        output.V2 = input.V1;
-        output.V3 = input.V3;
-
-        // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform.
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
-
-        // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range.
-        byte range = stageRange[3];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
-    }
-
-    /// <summary>
-    /// Applies the transform to four independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for the parallel transform axes.</param>
-    /// <param name="output">The destination values for the parallel transform axes.</param>
-    /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT.
-        output.V0 = input.V0;
-        output.V1 = input.V2;
-        output.V2 = input.V1;
-        output.V3 = input.V3;
-
-        // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform.
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
-        step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
-        step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
-        step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
-
-        // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range.
-        byte range = stageRange[3];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs

@@ -1,232 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the eight-point AV1 inverse discrete cosine transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
-/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
-/// </remarks>
-internal readonly struct Av1Dct8Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative eight-point AV1 inverse discrete cosine transform.
-    /// </summary>
-    /// <param name="input">The eight frequency-domain coefficients.</param>
-    /// <param name="output">The eight spatial-domain residual values.</param>
-    /// <param name="step">The eight-element stage buffer owned by the containing two-dimensional transform.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
-        stage++;
-        output[0] = input[0];
-        output[1] = input[4];
-        output[2] = input[2];
-        output[3] = input[6];
-        output[4] = input[1];
-        output[5] = input[5];
-        output[6] = input[3];
-        output[7] = input[7];
-
-        // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles.
-        stage++;
-        step[0] = output[0];
-        step[1] = output[1];
-        step[2] = output[2];
-        step[3] = output[3];
-        step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit);
-        step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit);
-        step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit);
-        step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit);
-
-        // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms.
-        stage++;
-        byte range = stageRange[stage];
-        output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit);
-        output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit);
-        output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit);
-        output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit);
-        output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range);
-        output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range);
-        output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range);
-        output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range);
-
-        // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation.
-        stage++;
-        step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range);
-        step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range);
-        step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range);
-        step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range);
-        step[4] = output[4];
-        step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit);
-        step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit);
-        step[7] = output[7];
-
-        // Stage 5 merges the even and odd halves into spatial order and clamps every result.
-        stage++;
-        range = stageRange[stage];
-        output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range);
-        output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range);
-        output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range);
-        output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range);
-        output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range);
-        output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range);
-        output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range);
-        output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range);
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
-        stage++;
-        output.V0 = input.V0;
-        output.V1 = input.V4;
-        output.V2 = input.V2;
-        output.V3 = input.V6;
-        output.V4 = input.V1;
-        output.V5 = input.V5;
-        output.V6 = input.V3;
-        output.V7 = input.V7;
-
-        // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit);
-
-        // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms.
-        stage++;
-        byte range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit);
-        output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit);
-        output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit);
-        output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range);
-        output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range);
-        output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range);
-        output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range);
-
-        // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation.
-        stage++;
-        step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range);
-        step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range);
-        step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range);
-        step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range);
-        step.V4 = output.V4;
-        step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit);
-        step.V7 = output.V7;
-
-        // Stage 5 merges the even and odd halves into spatial order and clamps every result.
-        stage++;
-        range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range);
-        output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range);
-        output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range);
-        output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range);
-        output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range);
-    }
-
-    /// <summary>
-    /// Applies the transform to four independent axes in parallel.
-    /// </summary>
-    /// <param name="input">The source values for the parallel transform axes.</param>
-    /// <param name="output">The destination values for the parallel transform axes.</param>
-    /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
-    /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
-    /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
-        int stage = 0;
-
-        // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
-        stage++;
-        output.V0 = input.V0;
-        output.V1 = input.V4;
-        output.V2 = input.V2;
-        output.V3 = input.V6;
-        output.V4 = input.V1;
-        output.V5 = input.V5;
-        output.V6 = input.V3;
-        output.V7 = input.V7;
-
-        // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles.
-        stage++;
-        step.V0 = output.V0;
-        step.V1 = output.V1;
-        step.V2 = output.V2;
-        step.V3 = output.V3;
-        step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit);
-        step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit);
-        step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit);
-
-        // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms.
-        stage++;
-        byte range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit);
-        output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit);
-        output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit);
-        output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit);
-        output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range);
-        output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range);
-        output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range);
-        output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range);
-
-        // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation.
-        stage++;
-        step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range);
-        step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range);
-        step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range);
-        step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range);
-        step.V4 = output.V4;
-        step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit);
-        step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit);
-        step.V7 = output.V7;
-
-        // Stage 5 merges the even and odd halves into spatial order and clamps every result.
-        stage++;
-        range = stageRange[stage];
-        output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range);
-        output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range);
-        output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range);
-        output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range);
-        output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range);
-        output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range);
-        output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range);
-        output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range);
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs

@@ -1,81 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the sixteen-point AV1 inverse identity transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
-/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
-/// </remarks>
-internal readonly struct Av1Identity16Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative sixteen-point AV1 inverse identity transform.
-    /// </summary>
-    /// <param name="input">The sixteen frequency-domain coefficients.</param>
-    /// <param name="output">The sixteen scaled spatial-domain values.</param>
-    /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
-    /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
-    /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-
-        // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization.
-        for (int i = 0; i < 16; i++)
-        {
-            output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits);
-        }
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and
-        // rounding sequence, matching libaom without changing the established lower-range SIMD path.
-        if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
-        {
-            Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-        else
-        {
-            Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-
-        _ = step;
-        _ = cosBit;
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
-        {
-            Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-        else
-        {
-            Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-
-        _ = step;
-        _ = cosBit;
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs

@@ -1,65 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the thirty-two-point AV1 inverse identity transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
-/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
-/// </remarks>
-internal readonly struct Av1Identity32Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative thirty-two-point AV1 inverse identity transform.
-    /// </summary>
-    /// <param name="input">The thirty-two frequency-domain coefficients.</param>
-    /// <param name="output">The thirty-two scaled spatial-domain values.</param>
-    /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
-    /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
-    /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-
-        // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization.
-        for (int i = 0; i < 32; i++)
-        {
-            output[i] = input[i] * 4;
-        }
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0);
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0);
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs

@@ -1,81 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the four-point AV1 inverse identity transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
-/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
-/// </remarks>
-internal readonly struct Av1Identity4Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative four-point AV1 inverse identity transform.
-    /// </summary>
-    /// <param name="input">The four frequency-domain coefficients.</param>
-    /// <param name="output">The four scaled spatial-domain values.</param>
-    /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
-    /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
-    /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-
-        // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization.
-        for (int i = 0; i < 4; i++)
-        {
-            output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product.
-        // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges.
-        if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
-        {
-            Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-        else
-        {
-            Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-
-        _ = step;
-        _ = cosBit;
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
-        {
-            Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-        else
-        {
-            Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
-        }
-
-        _ = step;
-        _ = cosBit;
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs

@@ -1,65 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
-
-/// <summary>
-/// Defines the eight-point AV1 inverse identity transform operator.
-/// </summary>
-/// <remarks>
-/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
-/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
-/// </remarks>
-internal readonly struct Av1Identity8Inverse1dOperator : IAv1Transform1dOperator
-{
-    /// <summary>
-    /// Applies the normative eight-point AV1 inverse identity transform.
-    /// </summary>
-    /// <param name="input">The eight frequency-domain coefficients.</param>
-    /// <param name="output">The eight scaled spatial-domain values.</param>
-    /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
-    /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
-    /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
-    public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
-    {
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-
-        // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization.
-        for (int i = 0; i < 8; i++)
-        {
-            output[i] = input[i] * 2;
-        }
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector128<int>> input,
-        ref Av1TransformVector<Vector128<int>> output,
-        ref Av1TransformVector<Vector128<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0);
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-    }
-
-    /// <inheritdoc/>
-    public static void Transform(
-        ref Av1TransformVector<Vector256<int>> input,
-        ref Av1TransformVector<Vector256<int>> output,
-        ref Av1TransformVector<Vector256<int>> step,
-        int cosBit,
-        Av1TransformStageRange stageRange)
-    {
-        Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0);
-        _ = step;
-        _ = cosBit;
-        _ = stageRange;
-    }
-}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst16Operator.cs

@@ -0,0 +1,571 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
+/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Adst16Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform.
+        /// </summary>
+        /// <param name="input">The sixteen frequency-domain coefficients.</param>
+        /// <param name="output">The sixteen spatial-domain residual values.</param>
+        /// <param name="step">The sixteen-element stage buffer owned by the containing two-dimensional transform.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
+            stage++;
+            output[0] = input[15];
+            output[1] = input[0];
+            output[2] = input[13];
+            output[3] = input[2];
+            output[4] = input[11];
+            output[5] = input[4];
+            output[6] = input[9];
+            output[7] = input[6];
+            output[8] = input[7];
+            output[9] = input[8];
+            output[10] = input[5];
+            output[11] = input[10];
+            output[12] = input[3];
+            output[13] = input[12];
+            output[14] = input[1];
+            output[15] = input[14];
+
+            // Stage 2 applies the terminal odd-angle rotations in reverse.
+            stage++;
+            step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit);
+            step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit);
+            step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit);
+            step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit);
+            step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit);
+            step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit);
+            step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit);
+            step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit);
+            step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit);
+            step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit);
+            step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit);
+            step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit);
+            step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit);
+            step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit);
+            step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit);
+
+            // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane.
+            stage++;
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]);
+            output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]);
+            output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]);
+            output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]);
+            output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]);
+            output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]);
+            output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]);
+            output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]);
+            output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]);
+            output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]);
+            output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]);
+            output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]);
+            output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]);
+            output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]);
+            output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]);
+
+            // Stage 4 reverses the pi/16 rotations in the upper half.
+            stage++;
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = output[2];
+            step[3] = output[3];
+            step[4] = output[4];
+            step[5] = output[5];
+            step[6] = output[6];
+            step[7] = output[7];
+            step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit);
+            step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit);
+            step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit);
+            step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit);
+            step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit);
+            step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit);
+            step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit);
+            step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit);
+
+            // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane.
+            stage++;
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]);
+            output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]);
+            output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]);
+            output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]);
+            output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]);
+            output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]);
+            output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]);
+            output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]);
+            output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]);
+            output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]);
+            output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]);
+            output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]);
+            output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]);
+            output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]);
+            output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]);
+
+            // Stage 6 reverses the pi/8 and 3pi/8 rotations.
+            stage++;
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = output[2];
+            step[3] = output[3];
+            step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit);
+            step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit);
+            step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit);
+            step[8] = output[8];
+            step[9] = output[9];
+            step[10] = output[10];
+            step[11] = output[11];
+            step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit);
+            step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit);
+            step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit);
+            step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit);
+
+            // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane.
+            stage++;
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]);
+            output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]);
+            output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]);
+            output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]);
+            output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]);
+            output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]);
+            output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]);
+            output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]);
+            output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]);
+            output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]);
+            output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]);
+            output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]);
+            output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]);
+            output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]);
+            output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]);
+
+            // Stage 8 reverses the pi/4 rotations for the middle pairs.
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit);
+            step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit);
+            step[4] = output[4];
+            step[5] = output[5];
+            step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit);
+            step[8] = output[8];
+            step[9] = output[9];
+            step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit);
+            step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit);
+            step[12] = output[12];
+            step[13] = output[13];
+            step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit);
+            step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit);
+
+            // Stage 9 applies the AV1 signs and permutation that restore spatial sample order.
+            output[0] = step[0];
+            output[1] = -step[8];
+            output[2] = step[12];
+            output[3] = -step[4];
+            output[4] = step[6];
+            output[5] = -step[14];
+            output[6] = step[10];
+            output[7] = -step[2];
+            output[8] = step[3];
+            output[9] = -step[11];
+            output[10] = step[15];
+            output[11] = -step[7];
+            output[12] = step[5];
+            output[13] = -step[13];
+            output[14] = step[9];
+            output[15] = -step[1];
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
+            stage++;
+            output.V0 = input.V15;
+            output.V1 = input.V0;
+            output.V2 = input.V13;
+            output.V3 = input.V2;
+            output.V4 = input.V11;
+            output.V5 = input.V4;
+            output.V6 = input.V9;
+            output.V7 = input.V6;
+            output.V8 = input.V7;
+            output.V9 = input.V8;
+            output.V10 = input.V5;
+            output.V11 = input.V10;
+            output.V12 = input.V3;
+            output.V13 = input.V12;
+            output.V14 = input.V1;
+            output.V15 = input.V14;
+
+            // Stage 2 applies the terminal odd-angle rotations in reverse.
+            stage++;
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit);
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit);
+            step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit);
+            step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit);
+
+            // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]);
+            output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]);
+            output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]);
+            output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]);
+            output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]);
+            output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]);
+            output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]);
+            output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]);
+            output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]);
+
+            // Stage 4 reverses the pi/16 rotations in the upper half.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = output.V6;
+            step.V7 = output.V7;
+            step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit);
+            step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit);
+
+            // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]);
+            output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]);
+            output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]);
+            output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]);
+            output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]);
+            output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]);
+            output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]);
+            output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]);
+
+            // Stage 6 reverses the pi/8 and 3pi/8 rotations.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
+            step.V8 = output.V8;
+            step.V9 = output.V9;
+            step.V10 = output.V10;
+            step.V11 = output.V11;
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit);
+
+            // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]);
+            output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]);
+            output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]);
+            output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]);
+            output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]);
+            output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]);
+            output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]);
+            output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]);
+
+            // Stage 8 reverses the pi/4 rotations for the middle pairs.
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
+            step.V8 = output.V8;
+            step.V9 = output.V9;
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit);
+            step.V12 = output.V12;
+            step.V13 = output.V13;
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit);
+
+            // Stage 9 applies the AV1 signs and permutation that restore spatial sample order.
+            output.V0 = step.V0;
+            output.V1 = -step.V8;
+            output.V2 = step.V12;
+            output.V3 = -step.V4;
+            output.V4 = step.V6;
+            output.V5 = -step.V14;
+            output.V6 = step.V10;
+            output.V7 = -step.V2;
+            output.V8 = step.V3;
+            output.V9 = -step.V11;
+            output.V10 = step.V15;
+            output.V11 = -step.V7;
+            output.V12 = step.V5;
+            output.V13 = -step.V13;
+            output.V14 = step.V9;
+            output.V15 = -step.V1;
+        }
+
+        /// <summary>
+        /// Applies the transform to four independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for the parallel transform axes.</param>
+        /// <param name="output">The destination values for the parallel transform axes.</param>
+        /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
+            stage++;
+            output.V0 = input.V15;
+            output.V1 = input.V0;
+            output.V2 = input.V13;
+            output.V3 = input.V2;
+            output.V4 = input.V11;
+            output.V5 = input.V4;
+            output.V6 = input.V9;
+            output.V7 = input.V6;
+            output.V8 = input.V7;
+            output.V9 = input.V8;
+            output.V10 = input.V5;
+            output.V11 = input.V10;
+            output.V12 = input.V3;
+            output.V13 = input.V12;
+            output.V14 = input.V1;
+            output.V15 = input.V14;
+
+            // Stage 2 applies the terminal odd-angle rotations in reverse.
+            stage++;
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit);
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit);
+            step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit);
+            step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit);
+
+            // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]);
+            output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]);
+            output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]);
+            output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]);
+            output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]);
+            output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]);
+            output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]);
+            output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]);
+            output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]);
+
+            // Stage 4 reverses the pi/16 rotations in the upper half.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = output.V6;
+            step.V7 = output.V7;
+            step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit);
+            step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit);
+
+            // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]);
+            output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]);
+            output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]);
+            output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]);
+            output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]);
+            output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]);
+            output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]);
+            output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]);
+
+            // Stage 6 reverses the pi/8 and 3pi/8 rotations.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
+            step.V8 = output.V8;
+            step.V9 = output.V9;
+            step.V10 = output.V10;
+            step.V11 = output.V11;
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit);
+
+            // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]);
+            output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]);
+            output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]);
+            output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]);
+            output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]);
+            output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]);
+            output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]);
+            output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]);
+
+            // Stage 8 reverses the pi/4 rotations for the middle pairs.
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
+            step.V8 = output.V8;
+            step.V9 = output.V9;
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit);
+            step.V12 = output.V12;
+            step.V13 = output.V13;
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit);
+
+            // Stage 9 applies the AV1 signs and permutation that restore spatial sample order.
+            output.V0 = step.V0;
+            output.V1 = -step.V8;
+            output.V2 = step.V12;
+            output.V3 = -step.V4;
+            output.V4 = step.V6;
+            output.V5 = -step.V14;
+            output.V6 = step.V10;
+            output.V7 = -step.V2;
+            output.V8 = step.V3;
+            output.V9 = -step.V11;
+            output.V10 = step.V15;
+            output.V11 = -step.V7;
+            output.V12 = step.V5;
+            output.V13 = -step.V13;
+            output.V14 = step.V9;
+            output.V15 = -step.V1;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst4Operator.cs

@@ -0,0 +1,145 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
+/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Adst4Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform.
+        /// </summary>
+        /// <param name="input">The four frequency-domain coefficients.</param>
+        /// <param name="output">The four spatial-domain residual values.</param>
+        /// <param name="step">The stage buffer owned by the containing two-dimensional transform.</param>
+        /// <param name="cosBit">The fixed-point precision of the sine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+
+            // libaom widens the complete four-point factorization because the products retain their fixed-point scale
+            // until the final shift. The stage buffer is therefore unnecessary for this transform size.
+            long x0 = input[0];
+            long x1 = input[1];
+            long x2 = input[2];
+            long x3 = input[3];
+
+            _ = step;
+            _ = stageRange;
+
+            // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel.
+            if ((x0 | x1 | x2 | x3) == 0)
+            {
+                output[..4].Clear();
+                return;
+            }
+
+            // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3.
+            long s0 = sinpi[1] * x0;
+            long s1 = sinpi[2] * x0;
+            long s2 = sinpi[3] * x1;
+            long s3 = sinpi[4] * x2;
+            long s4 = sinpi[1] * x2;
+            long s5 = sinpi[2] * x3;
+            long s6 = sinpi[4] * x3;
+            long s7 = (x0 - x2) + x3;
+
+            // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding.
+            s0 += s3;
+            s1 -= s4;
+            s3 = s2;
+            s2 = sinpi[3] * s7;
+            s0 += s5;
+            s1 -= s6;
+            x0 = s0 + s3;
+            x1 = s1 + s3;
+            x2 = s2;
+            x3 = (s0 + s1) - s3;
+
+            output[0] = Av1Math.RoundShift(x0, cosBit);
+            output[1] = Av1Math.RoundShift(x1, cosBit);
+            output[2] = Av1Math.RoundShift(x2, cosBit);
+            output[3] = Av1Math.RoundShift(x3, cosBit);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            bool widenedRound = stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Vector128<int> x0 = input.V0;
+            Vector128<int> x1 = input.V1;
+            Vector128<int> x2 = input.V2;
+            Vector128<int> x3 = input.V3;
+
+            // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row
+            // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow
+            // a valid Int32 fixed-point sum.
+            if (widenedRound)
+            {
+                output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
+                output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
+                output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
+                output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
+                return;
+            }
+
+            output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
+            output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
+            output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
+            output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
+
+            _ = step;
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            bool widenedRound = stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount;
+
+            ReadOnlySpan<int> sinpi = Av1SinusConstants.SinusPi(cosBit);
+            Vector256<int> x0 = input.V0;
+            Vector256<int> x1 = input.V1;
+            Vector256<int> x2 = input.V2;
+            Vector256<int> x3 = input.V3;
+
+            if (widenedRound)
+            {
+                output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
+                output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
+                output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
+                output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
+                return;
+            }
+
+            output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit);
+            output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit);
+            output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit);
+            output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit);
+
+            _ = step;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst8Operator.cs

@@ -0,0 +1,292 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
+/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Adst8Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform.
+        /// </summary>
+        /// <param name="input">The eight frequency-domain coefficients.</param>
+        /// <param name="output">The eight spatial-domain residual values.</param>
+        /// <param name="step">The eight-element stage buffer owned by the containing two-dimensional transform.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
+            stage++;
+            output[0] = input[7];
+            output[1] = input[0];
+            output[2] = input[5];
+            output[3] = input[2];
+            output[4] = input[3];
+            output[5] = input[4];
+            output[6] = input[1];
+            output[7] = input[6];
+
+            // Stage 2 applies the terminal odd-angle rotations in reverse.
+            stage++;
+            step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit);
+            step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit);
+            step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit);
+            step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit);
+            step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit);
+            step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit);
+            step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit);
+
+            // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane.
+            stage++;
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]);
+            output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]);
+            output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]);
+            output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]);
+            output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]);
+            output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]);
+            output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]);
+
+            // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half.
+            stage++;
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = output[2];
+            step[3] = output[3];
+            step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit);
+            step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit);
+            step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit);
+
+            // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane.
+            stage++;
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]);
+            output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]);
+            output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]);
+            output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]);
+            output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]);
+            output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]);
+            output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]);
+
+            // Stage 6 reverses the pi/4 rotations for the middle pairs.
+            stage++;
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit);
+            step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit);
+            step[4] = output[4];
+            step[5] = output[5];
+            step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit);
+
+            // Stage 7 applies the AV1 signs and permutation that restore spatial sample order.
+            output[0] = step[0];
+            output[1] = -step[4];
+            output[2] = step[6];
+            output[3] = -step[2];
+            output[4] = step[3];
+            output[5] = -step[7];
+            output[6] = step[5];
+            output[7] = -step[1];
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
+            stage++;
+            output.V0 = input.V7;
+            output.V1 = input.V0;
+            output.V2 = input.V5;
+            output.V3 = input.V2;
+            output.V4 = input.V3;
+            output.V5 = input.V4;
+            output.V6 = input.V1;
+            output.V7 = input.V6;
+
+            // Stage 2 applies the terminal odd-angle rotations in reverse.
+            stage++;
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit);
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit);
+
+            // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
+
+            // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
+
+            // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
+
+            // Stage 6 reverses the pi/4 rotations for the middle pairs.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
+
+            // Stage 7 applies the AV1 signs and permutation that restore spatial sample order.
+            output.V0 = step.V0;
+            output.V1 = -step.V4;
+            output.V2 = step.V6;
+            output.V3 = -step.V2;
+            output.V4 = step.V3;
+            output.V5 = -step.V7;
+            output.V6 = step.V5;
+            output.V7 = -step.V1;
+        }
+
+        /// <summary>
+        /// Applies the transform to four independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for the parallel transform axes.</param>
+        /// <param name="output">The destination values for the parallel transform axes.</param>
+        /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes the coefficients into the signed order used by the ADST factorization.
+            stage++;
+            output.V0 = input.V7;
+            output.V1 = input.V0;
+            output.V2 = input.V5;
+            output.V3 = input.V2;
+            output.V4 = input.V3;
+            output.V5 = input.V4;
+            output.V6 = input.V1;
+            output.V7 = input.V6;
+
+            // Stage 2 applies the terminal odd-angle rotations in reverse.
+            stage++;
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit);
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit);
+
+            // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]);
+
+            // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit);
+
+            // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane.
+            stage++;
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]);
+            output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]);
+            output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]);
+            output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]);
+            output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]);
+
+            // Stage 6 reverses the pi/4 rotations for the middle pairs.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit);
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit);
+
+            // Stage 7 applies the AV1 signs and permutation that restore spatial sample order.
+            output.V0 = step.V0;
+            output.V1 = -step.V4;
+            output.V2 = step.V6;
+            output.V3 = -step.V2;
+            output.V4 = step.V3;
+            output.V5 = -step.V7;
+            output.V6 = step.V5;
+            output.V7 = -step.V1;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct16Operator.cs

@@ -0,0 +1,478 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the 16-point AV1 inverse discrete cosine transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
+/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Dct16Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative 16-point AV1 inverse discrete cosine transform.
+        /// </summary>
+        /// <param name="input">The sixteen frequency-domain coefficients.</param>
+        /// <param name="output">The sixteen spatial-domain residual values.</param>
+        /// <param name="step">The sixteen-element stage buffer owned by the containing two-dimensional transform.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
+            stage++;
+            output[0] = input[0];
+            output[1] = input[8];
+            output[2] = input[4];
+            output[3] = input[12];
+            output[4] = input[2];
+            output[5] = input[10];
+            output[6] = input[6];
+            output[7] = input[14];
+            output[8] = input[1];
+            output[9] = input[9];
+            output[10] = input[5];
+            output[11] = input[13];
+            output[12] = input[3];
+            output[13] = input[11];
+            output[14] = input[7];
+            output[15] = input[15];
+
+            // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles.
+            stage++;
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = output[2];
+            step[3] = output[3];
+            step[4] = output[4];
+            step[5] = output[5];
+            step[6] = output[6];
+            step[7] = output[7];
+            step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit);
+            step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit);
+            step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit);
+            step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit);
+            step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit);
+            step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit);
+            step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit);
+            step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit);
+
+            // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms.
+            stage++;
+            byte range = stageRange[stage];
+            output[0] = step[0];
+            output[1] = step[1];
+            output[2] = step[2];
+            output[3] = step[3];
+            output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit);
+            output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit);
+            output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit);
+            output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit);
+            output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range);
+            output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range);
+            output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range);
+            output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range);
+            output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range);
+            output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range);
+            output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range);
+            output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range);
+
+            // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs.
+            stage++;
+            range = stageRange[stage];
+            step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit);
+            step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit);
+            step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit);
+            step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit);
+            step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range);
+            step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range);
+            step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range);
+            step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range);
+            step[8] = output[8];
+            step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit);
+            step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit);
+            step[11] = output[11];
+            step[12] = output[12];
+            step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit);
+            step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit);
+            step[15] = output[15];
+
+            // Stage 5 widens the reconstructed groups through their next butterfly level.
+            stage++;
+            range = stageRange[stage];
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range);
+            output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range);
+            output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range);
+            output[4] = step[4];
+            output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit);
+            output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit);
+            output[7] = step[7];
+            output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range);
+            output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range);
+            output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range);
+            output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range);
+            output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range);
+            output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range);
+            output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range);
+            output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range);
+
+            // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge.
+            stage++;
+            range = stageRange[stage];
+            step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range);
+            step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range);
+            step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range);
+            step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range);
+            step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range);
+            step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range);
+            step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range);
+            step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range);
+            step[8] = output[8];
+            step[9] = output[9];
+            step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit);
+            step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit);
+            step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit);
+            step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit);
+            step[14] = output[14];
+            step[15] = output[15];
+
+            // Stage 7 merges the even and odd halves into spatial order and clamps every result.
+            stage++;
+            range = stageRange[stage];
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range);
+            output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range);
+            output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range);
+            output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range);
+            output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range);
+            output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range);
+            output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range);
+            output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range);
+            output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range);
+            output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range);
+            output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range);
+            output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range);
+            output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range);
+            output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range);
+            output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
+            stage++;
+            output.V0 = input.V0;
+            output.V1 = input.V8;
+            output.V2 = input.V4;
+            output.V3 = input.V12;
+            output.V4 = input.V2;
+            output.V5 = input.V10;
+            output.V6 = input.V6;
+            output.V7 = input.V14;
+            output.V8 = input.V1;
+            output.V9 = input.V9;
+            output.V10 = input.V5;
+            output.V11 = input.V13;
+            output.V12 = input.V3;
+            output.V13 = input.V11;
+            output.V14 = input.V7;
+            output.V15 = input.V15;
+
+            // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = output.V6;
+            step.V7 = output.V7;
+            step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit);
+            step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit);
+
+            // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms.
+            stage++;
+            byte range = stageRange[stage];
+            output.V0 = step.V0;
+            output.V1 = step.V1;
+            output.V2 = step.V2;
+            output.V3 = step.V3;
+            output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit);
+            output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit);
+            output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit);
+            output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit);
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range);
+            output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range);
+            output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range);
+            output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range);
+            output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range);
+            output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range);
+            output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range);
+            output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range);
+
+            // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs.
+            stage++;
+            range = stageRange[stage];
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
+            step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range);
+            step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range);
+            step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range);
+            step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range);
+            step.V8 = output.V8;
+            step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit);
+            step.V11 = output.V11;
+            step.V12 = output.V12;
+            step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit);
+            step.V15 = output.V15;
+
+            // Stage 5 widens the reconstructed groups through their next butterfly level.
+            stage++;
+            range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
+            output.V4 = step.V4;
+            output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit);
+            output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit);
+            output.V7 = step.V7;
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range);
+            output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range);
+            output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range);
+            output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range);
+            output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range);
+            output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range);
+            output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range);
+            output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range);
+
+            // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge.
+            stage++;
+            range = stageRange[stage];
+            step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range);
+            step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range);
+            step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range);
+            step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range);
+            step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range);
+            step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range);
+            step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range);
+            step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range);
+            step.V8 = output.V8;
+            step.V9 = output.V9;
+            step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit);
+            step.V14 = output.V14;
+            step.V15 = output.V15;
+
+            // Stage 7 merges the even and odd halves into spatial order and clamps every result.
+            stage++;
+            range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range);
+            output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range);
+            output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range);
+            output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range);
+            output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range);
+            output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range);
+            output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range);
+            output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range);
+            output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range);
+            output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range);
+            output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range);
+        }
+
+        /// <summary>
+        /// Applies the transform to four independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for the parallel transform axes.</param>
+        /// <param name="output">The destination values for the parallel transform axes.</param>
+        /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
+            stage++;
+            output.V0 = input.V0;
+            output.V1 = input.V8;
+            output.V2 = input.V4;
+            output.V3 = input.V12;
+            output.V4 = input.V2;
+            output.V5 = input.V10;
+            output.V6 = input.V6;
+            output.V7 = input.V14;
+            output.V8 = input.V1;
+            output.V9 = input.V9;
+            output.V10 = input.V5;
+            output.V11 = input.V13;
+            output.V12 = input.V3;
+            output.V13 = input.V11;
+            output.V14 = input.V7;
+            output.V15 = input.V15;
+
+            // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = output.V4;
+            step.V5 = output.V5;
+            step.V6 = output.V6;
+            step.V7 = output.V7;
+            step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit);
+            step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit);
+            step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit);
+
+            // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms.
+            stage++;
+            byte range = stageRange[stage];
+            output.V0 = step.V0;
+            output.V1 = step.V1;
+            output.V2 = step.V2;
+            output.V3 = step.V3;
+            output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit);
+            output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit);
+            output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit);
+            output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit);
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range);
+            output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range);
+            output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range);
+            output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range);
+            output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range);
+            output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range);
+            output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range);
+            output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range);
+
+            // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs.
+            stage++;
+            range = stageRange[stage];
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
+            step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range);
+            step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range);
+            step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range);
+            step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range);
+            step.V8 = output.V8;
+            step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit);
+            step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit);
+            step.V11 = output.V11;
+            step.V12 = output.V12;
+            step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit);
+            step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit);
+            step.V15 = output.V15;
+
+            // Stage 5 widens the reconstructed groups through their next butterfly level.
+            stage++;
+            range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
+            output.V4 = step.V4;
+            output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit);
+            output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit);
+            output.V7 = step.V7;
+            output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range);
+            output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range);
+            output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range);
+            output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range);
+            output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range);
+            output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range);
+            output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range);
+            output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range);
+
+            // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge.
+            stage++;
+            range = stageRange[stage];
+            step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range);
+            step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range);
+            step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range);
+            step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range);
+            step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range);
+            step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range);
+            step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range);
+            step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range);
+            step.V8 = output.V8;
+            step.V9 = output.V9;
+            step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit);
+            step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit);
+            step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit);
+            step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit);
+            step.V14 = output.V14;
+            step.V15 = output.V15;
+
+            // Stage 7 merges the even and odd halves into spatial order and clamps every result.
+            stage++;
+            range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range);
+            output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range);
+            output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range);
+            output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range);
+            output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range);
+            output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range);
+            output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range);
+            output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range);
+            output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range);
+            output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range);
+            output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range);
+            output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct4Operator.cs

@@ -0,0 +1,115 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the four-point AV1 inverse discrete cosine transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
+/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Dct4Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative four-point AV1 inverse discrete cosine transform.
+        /// </summary>
+        /// <param name="input">The four frequency-domain coefficients.</param>
+        /// <param name="output">The four spatial-domain residual values.</param>
+        /// <param name="step">The four-element stage buffer owned by the containing two-dimensional transform.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT.
+            output[0] = input[0];
+            output[1] = input[2];
+            output[2] = input[1];
+            output[3] = input[3];
+
+            // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform.
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit);
+            step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit);
+            step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit);
+            step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit);
+
+            // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range.
+            byte range = stageRange[3];
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range);
+            output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range);
+            output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT.
+            output.V0 = input.V0;
+            output.V1 = input.V2;
+            output.V2 = input.V1;
+            output.V3 = input.V3;
+
+            // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform.
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
+
+            // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range.
+            byte range = stageRange[3];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
+        }
+
+        /// <summary>
+        /// Applies the transform to four independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for the parallel transform axes.</param>
+        /// <param name="output">The destination values for the parallel transform axes.</param>
+        /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT.
+            output.V0 = input.V0;
+            output.V1 = input.V2;
+            output.V2 = input.V1;
+            output.V3 = input.V3;
+
+            // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform.
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit);
+            step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit);
+            step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit);
+            step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit);
+
+            // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range.
+            byte range = stageRange[3];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct8Operator.cs

@@ -0,0 +1,235 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the eight-point AV1 inverse discrete cosine transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
+/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Dct8Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative eight-point AV1 inverse discrete cosine transform.
+        /// </summary>
+        /// <param name="input">The eight frequency-domain coefficients.</param>
+        /// <param name="output">The eight spatial-domain residual values.</param>
+        /// <param name="step">The eight-element stage buffer owned by the containing two-dimensional transform.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
+            stage++;
+            output[0] = input[0];
+            output[1] = input[4];
+            output[2] = input[2];
+            output[3] = input[6];
+            output[4] = input[1];
+            output[5] = input[5];
+            output[6] = input[3];
+            output[7] = input[7];
+
+            // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles.
+            stage++;
+            step[0] = output[0];
+            step[1] = output[1];
+            step[2] = output[2];
+            step[3] = output[3];
+            step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit);
+            step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit);
+            step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit);
+            step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit);
+
+            // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms.
+            stage++;
+            byte range = stageRange[stage];
+            output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit);
+            output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit);
+            output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit);
+            output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit);
+            output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range);
+            output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range);
+            output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range);
+            output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range);
+
+            // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation.
+            stage++;
+            step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range);
+            step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range);
+            step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range);
+            step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range);
+            step[4] = output[4];
+            step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit);
+            step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit);
+            step[7] = output[7];
+
+            // Stage 5 merges the even and odd halves into spatial order and clamps every result.
+            stage++;
+            range = stageRange[stage];
+            output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range);
+            output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range);
+            output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range);
+            output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range);
+            output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range);
+            output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range);
+            output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range);
+            output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range);
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
+            stage++;
+            output.V0 = input.V0;
+            output.V1 = input.V4;
+            output.V2 = input.V2;
+            output.V3 = input.V6;
+            output.V4 = input.V1;
+            output.V5 = input.V5;
+            output.V6 = input.V3;
+            output.V7 = input.V7;
+
+            // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit);
+
+            // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms.
+            stage++;
+            byte range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit);
+            output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit);
+            output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit);
+            output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range);
+            output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range);
+            output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range);
+            output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range);
+
+            // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation.
+            stage++;
+            step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range);
+            step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range);
+            step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range);
+            step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range);
+            step.V4 = output.V4;
+            step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit);
+            step.V7 = output.V7;
+
+            // Stage 5 merges the even and odd halves into spatial order and clamps every result.
+            stage++;
+            range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range);
+            output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range);
+            output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range);
+            output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range);
+            output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range);
+        }
+
+        /// <summary>
+        /// Applies the transform to four independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for the parallel transform axes.</param>
+        /// <param name="output">The destination values for the parallel transform axes.</param>
+        /// <param name="step">The fixed stage storage for the parallel transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            ReadOnlySpan<int> cospi = Av1SinusConstants.CosinusPi(cosBit);
+            int stage = 0;
+
+            // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order.
+            stage++;
+            output.V0 = input.V0;
+            output.V1 = input.V4;
+            output.V2 = input.V2;
+            output.V3 = input.V6;
+            output.V4 = input.V1;
+            output.V5 = input.V5;
+            output.V6 = input.V3;
+            output.V7 = input.V7;
+
+            // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles.
+            stage++;
+            step.V0 = output.V0;
+            step.V1 = output.V1;
+            step.V2 = output.V2;
+            step.V3 = output.V3;
+            step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit);
+            step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit);
+            step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit);
+
+            // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms.
+            stage++;
+            byte range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit);
+            output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit);
+            output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit);
+            output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit);
+            output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range);
+            output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range);
+            output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range);
+            output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range);
+
+            // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation.
+            stage++;
+            step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range);
+            step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range);
+            step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range);
+            step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range);
+            step.V4 = output.V4;
+            step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit);
+            step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit);
+            step.V7 = output.V7;
+
+            // Stage 5 merges the even and odd halves into spatial order and clamps every result.
+            stage++;
+            range = stageRange[stage];
+            output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range);
+            output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range);
+            output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range);
+            output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range);
+            output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range);
+            output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range);
+            output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range);
+            output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range);
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity16Operator.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the sixteen-point AV1 inverse identity transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
+/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Identity16Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative sixteen-point AV1 inverse identity transform.
+        /// </summary>
+        /// <param name="input">The sixteen frequency-domain coefficients.</param>
+        /// <param name="output">The sixteen scaled spatial-domain values.</param>
+        /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
+        /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
+        /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+
+            // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization.
+            for (int i = 0; i < 16; i++)
+            {
+                output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and
+            // rounding sequence, matching libaom without changing the established lower-range SIMD path.
+            if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
+            {
+                Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+            else
+            {
+                Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+
+            _ = step;
+            _ = cosBit;
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
+            {
+                Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+            else
+            {
+                Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+
+            _ = step;
+            _ = cosBit;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity32Operator.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the thirty-two-point AV1 inverse identity transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
+/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Identity32Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative thirty-two-point AV1 inverse identity transform.
+        /// </summary>
+        /// <param name="input">The thirty-two frequency-domain coefficients.</param>
+        /// <param name="output">The thirty-two scaled spatial-domain values.</param>
+        /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
+        /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
+        /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+
+            // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization.
+            for (int i = 0; i < 32; i++)
+            {
+                output[i] = input[i] * 4;
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0);
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0);
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity4Operator.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the four-point AV1 inverse identity transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
+/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Identity4Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative four-point AV1 inverse identity transform.
+        /// </summary>
+        /// <param name="input">The four frequency-domain coefficients.</param>
+        /// <param name="output">The four scaled spatial-domain values.</param>
+        /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
+        /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
+        /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+
+            // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization.
+            for (int i = 0; i < 4; i++)
+            {
+                output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product.
+            // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges.
+            if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
+            {
+                Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+            else
+            {
+                Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+
+            _ = step;
+            _ = cosBit;
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount)
+            {
+                Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+            else
+            {
+                Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits);
+            }
+
+            _ = step;
+            _ = cosBit;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity8Operator.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <summary>
+/// Defines the eight-point AV1 inverse identity transform operator.
+/// </summary>
+/// <remarks>
+/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
+/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
+/// </remarks>
+internal static partial class Av1Inverse2dTransformer
+{
+    internal readonly struct Identity8Operator : IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Applies the normative eight-point AV1 inverse identity transform.
+        /// </summary>
+        /// <param name="input">The eight frequency-domain coefficients.</param>
+        /// <param name="output">The eight scaled spatial-domain values.</param>
+        /// <param name="step">Unused stage storage supplied by the common transform-kernel contract.</param>
+        /// <param name="cosBit">Unused cosine precision supplied by the common transform-kernel contract.</param>
+        /// <param name="stageRange">The signed-bit range assigned to the transform output.</param>
+        public static void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange)
+        {
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+
+            // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization.
+            for (int i = 0; i < 8; i++)
+            {
+                output[i] = input[i] * 2;
+            }
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0);
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+        }
+
+        /// <inheritdoc/>
+        public static void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange)
+        {
+            Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0);
+            _ = step;
+            _ = cosBit;
+            _ = stageRange;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Operator.cs

@@ -0,0 +1,62 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+/// <content>
+/// Defines the inverse-transform operator contract.
+/// </content>
+internal static partial class Av1Inverse2dTransformer
+{
+    /// <summary>
+    /// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional inverse transform.
+    /// </summary>
+    /// <remarks>
+    /// Each overload performs the same staged fixed-point transform. Vector fields identify coefficient positions,
+    /// while vector lanes identify independent rows or columns.
+    /// </remarks>
+    internal interface IAv1Transform1dOperator
+    {
+        /// <summary>
+        /// Transforms one axis when hardware vectorization is unavailable.
+        /// </summary>
+        /// <param name="input">The source values for the transform axis.</param>
+        /// <param name="output">The destination values for the transform axis.</param>
+        /// <param name="step">The fixed stage storage for the transform axis.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static abstract void Transform(ReadOnlySpan<int> input, Span<int> output, Span<int> step, int cosBit, Av1TransformStageRange stageRange);
+
+        /// <summary>
+        /// Transforms four independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for four transform axes.</param>
+        /// <param name="output">The destination values for four transform axes.</param>
+        /// <param name="step">The fixed stage storage for four transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static abstract void Transform(
+            ref Av1TransformVector<Vector128<int>> input,
+            ref Av1TransformVector<Vector128<int>> output,
+            ref Av1TransformVector<Vector128<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange);
+
+        /// <summary>
+        /// Transforms eight independent axes in parallel.
+        /// </summary>
+        /// <param name="input">The source values for eight transform axes.</param>
+        /// <param name="output">The destination values for eight transform axes.</param>
+        /// <param name="step">The fixed stage storage for eight transform axes.</param>
+        /// <param name="cosBit">The fixed-point precision of the cosine constants.</param>
+        /// <param name="stageRange">The signed-bit range assigned to each transform stage.</param>
+        public static abstract void Transform(
+            ref Av1TransformVector<Vector256<int>> input,
+            ref Av1TransformVector<Vector256<int>> output,
+            ref Av1TransformVector<Vector256<int>> step,
+            int cosBit,
+            Av1TransformStageRange stageRange);
+    }
+}

src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.Operator.cs → src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.VectorOperations.cs

@@ -3,13 +3,13 @@
 
 using System.Runtime.Intrinsics;
 using SixLabors.ImageSharp.Metadata.Profiles.Cicp;
-using static SixLabors.ImageSharp.Formats.Heif.Components.HeifTransferVectorOperators;
+using static SixLabors.ImageSharp.Formats.Heif.Components.HeifTransferVectorOperations;
 
 namespace SixLabors.ImageSharp.Formats.Heif.Components;
 
 /// <content>
 /// Provides fixed-width vector overloads and shared H.273 transfer operations for HEIF color conversion. One lane
-/// represents one normalized color component. Closed vector operators bind the 128-, 256-, or 512-bit implementation
+/// represents one normalized color component. Closed vector operations implementations bind the 128-, 256-, or 512-bit implementation
 /// once per row kernel, while conditional selection evaluates piecewise transfer curves without per-lane branches.
 /// Inputs to logarithms and powers are bounded before evaluation because SIMD selection evaluates both branches.
 /// </content>
@@ -22,7 +22,7 @@ internal static partial class HeifTransferFunctions
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The corresponding linear-domain values.</returns>
     public static Vector128<float> ToLinear(CicpTransferCharacteristics transferCharacteristics, Vector128<float> value)
-        => ToLinear<Vector128<float>, Vector128Operator>(transferCharacteristics, value);
+        => ToLinear<Vector128<float>, Vector128Operations>(transferCharacteristics, value);
 
     /// <summary>
     /// Converts eight nonlinear signal values to their H.273 linear-domain values.
@@ -31,7 +31,7 @@ internal static partial class HeifTransferFunctions
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The corresponding linear-domain values.</returns>
     public static Vector256<float> ToLinear(CicpTransferCharacteristics transferCharacteristics, Vector256<float> value)
-        => ToLinear<Vector256<float>, Vector256Operator>(transferCharacteristics, value);
+        => ToLinear<Vector256<float>, Vector256Operations>(transferCharacteristics, value);
 
     /// <summary>
     /// Converts sixteen nonlinear signal values to their H.273 linear-domain values.
@@ -40,7 +40,7 @@ internal static partial class HeifTransferFunctions
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The corresponding linear-domain values.</returns>
     public static Vector512<float> ToLinear(CicpTransferCharacteristics transferCharacteristics, Vector512<float> value)
-        => ToLinear<Vector512<float>, Vector512Operator>(transferCharacteristics, value);
+        => ToLinear<Vector512<float>, Vector512Operations>(transferCharacteristics, value);
 
     /// <summary>
     /// Converts four linear signal values to their H.273 nonlinear-domain values.
@@ -49,7 +49,7 @@ internal static partial class HeifTransferFunctions
     /// <param name="value">The linear signal values.</param>
     /// <returns>The corresponding nonlinear-domain values.</returns>
     public static Vector128<float> ToGamma(CicpTransferCharacteristics transferCharacteristics, Vector128<float> value)
-        => ToGamma<Vector128<float>, Vector128Operator>(transferCharacteristics, value);
+        => ToGamma<Vector128<float>, Vector128Operations>(transferCharacteristics, value);
 
     /// <summary>
     /// Converts eight linear signal values to their H.273 nonlinear-domain values.
@@ -58,7 +58,7 @@ internal static partial class HeifTransferFunctions
     /// <param name="value">The linear signal values.</param>
     /// <returns>The corresponding nonlinear-domain values.</returns>
     public static Vector256<float> ToGamma(CicpTransferCharacteristics transferCharacteristics, Vector256<float> value)
-        => ToGamma<Vector256<float>, Vector256Operator>(transferCharacteristics, value);
+        => ToGamma<Vector256<float>, Vector256Operations>(transferCharacteristics, value);
 
     /// <summary>
     /// Converts sixteen linear signal values to their H.273 nonlinear-domain values.
@@ -67,22 +67,22 @@ internal static partial class HeifTransferFunctions
     /// <param name="value">The linear signal values.</param>
     /// <returns>The corresponding nonlinear-domain values.</returns>
     public static Vector512<float> ToGamma(CicpTransferCharacteristics transferCharacteristics, Vector512<float> value)
-        => ToGamma<Vector512<float>, Vector512Operator>(transferCharacteristics, value);
+        => ToGamma<Vector512<float>, Vector512Operations>(transferCharacteristics, value);
 
     /// <summary>
     /// Converts nonlinear signal values to their H.273 linear-domain values using the selected SIMD width.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="transferCharacteristics">The signaled transfer characteristics.</param>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The corresponding linear-domain values.</returns>
-    private static TVector ToLinear<TVector, TOperator>(CicpTransferCharacteristics transferCharacteristics, TVector value)
+    private static TVector ToLinear<TVector, TOperations>(CicpTransferCharacteristics transferCharacteristics, TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector one = TOperator.Create(1F);
+        TVector zero = TOperations.Create(0F);
+        TVector one = TOperations.Create(1F);
 
         switch (transferCharacteristics)
         {
@@ -90,45 +90,45 @@ internal static partial class HeifTransferFunctions
             case CicpTransferCharacteristics.ItuRBt601_7:
             case CicpTransferCharacteristics.ItuRBt2020_2_10bit:
             case CicpTransferCharacteristics.ItuRBt2020_2_12bit:
-                return ToLinearBt709<TVector, TOperator>(value);
+                return ToLinearBt709<TVector, TOperations>(value);
             case CicpTransferCharacteristics.Gamma2_2:
-                return Power<TVector, TOperator>(TOperator.Min(TOperator.Max(value, zero), one), 2.2F);
+                return Power<TVector, TOperations>(TOperations.Min(TOperations.Max(value, zero), one), 2.2F);
             case CicpTransferCharacteristics.Gamma2_8:
-                return Power<TVector, TOperator>(TOperator.Min(TOperator.Max(value, zero), one), 2.8F);
+                return Power<TVector, TOperations>(TOperations.Min(TOperations.Max(value, zero), one), 2.8F);
             case CicpTransferCharacteristics.SmpteSt240:
-                return ToLinearSmpte240<TVector, TOperator>(value);
+                return ToLinearSmpte240<TVector, TOperations>(value);
             case CicpTransferCharacteristics.Linear:
-                return TOperator.Min(TOperator.Max(value, zero), one);
+                return TOperations.Min(TOperations.Max(value, zero), one);
             case CicpTransferCharacteristics.Log100:
             {
                 // H.273 assigns an interval to zero for logarithmic curves. The scalar midpoint convention is
                 // selected lane-wise after evaluating the positive branch, which keeps the hot path branchless.
-                TVector exponent = TOperator.Multiply(TOperator.Subtract(TOperator.Min(value, one), one), TOperator.Create(2F * 2.302585092994046F));
-                TVector positive = TOperator.Exp(exponent);
-                return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), TOperator.Create(0.005F), positive);
+                TVector exponent = TOperations.Multiply(TOperations.Subtract(TOperations.Min(value, one), one), TOperations.Create(2F * 2.302585092994046F));
+                TVector positive = TOperations.Exp(exponent);
+                return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), TOperations.Create(0.005F), positive);
             }
 
             case CicpTransferCharacteristics.Log100Sqrt:
             {
-                TVector exponent = TOperator.Multiply(TOperator.Subtract(TOperator.Min(value, one), one), TOperator.Create(2.5F * 2.302585092994046F));
-                TVector positive = TOperator.Exp(exponent);
-                return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), TOperator.Create(0.00158113883F), positive);
+                TVector exponent = TOperations.Multiply(TOperations.Subtract(TOperations.Min(value, one), one), TOperations.Create(2.5F * 2.302585092994046F));
+                TVector positive = TOperations.Exp(exponent);
+                return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), TOperations.Create(0.00158113883F), positive);
             }
 
             case CicpTransferCharacteristics.Iec61966_2_4:
-                return ToLinearIec61966<TVector, TOperator>(value);
+                return ToLinearIec61966<TVector, TOperations>(value);
             case CicpTransferCharacteristics.ItuRBt1361_0:
-                return ToLinearBt1361<TVector, TOperator>(value);
+                return ToLinearBt1361<TVector, TOperations>(value);
             case CicpTransferCharacteristics.Iec61966_2_1:
-                return ToLinearSrgb<TVector, TOperator>(value);
+                return ToLinearSrgb<TVector, TOperations>(value);
             case CicpTransferCharacteristics.SmpteSt2084:
-                return ToLinearPq<TVector, TOperator>(value);
+                return ToLinearPq<TVector, TOperations>(value);
             case CicpTransferCharacteristics.SmpteSt428_1:
-                return TOperator.Divide(Power<TVector, TOperator>(TOperator.Max(value, zero), 2.6F), TOperator.Create(Smpte428Scale));
+                return TOperations.Divide(Power<TVector, TOperations>(TOperations.Max(value, zero), 2.6F), TOperations.Create(Smpte428Scale));
             case CicpTransferCharacteristics.AribStdB67:
-                return ToLinearHlg<TVector, TOperator>(value);
+                return ToLinearHlg<TVector, TOperations>(value);
             default:
-                return ToLinearBt709<TVector, TOperator>(value);
+                return ToLinearBt709<TVector, TOperations>(value);
         }
     }
 
@@ -136,16 +136,16 @@ internal static partial class HeifTransferFunctions
     /// Converts linear signal values to their H.273 nonlinear-domain values using the selected SIMD width.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="transferCharacteristics">The signaled transfer characteristics.</param>
     /// <param name="value">The linear signal values.</param>
     /// <returns>The corresponding nonlinear-domain values.</returns>
-    private static TVector ToGamma<TVector, TOperator>(CicpTransferCharacteristics transferCharacteristics, TVector value)
+    private static TVector ToGamma<TVector, TOperations>(CicpTransferCharacteristics transferCharacteristics, TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector one = TOperator.Create(1F);
+        TVector zero = TOperations.Create(0F);
+        TVector one = TOperations.Create(1F);
 
         switch (transferCharacteristics)
         {
@@ -153,47 +153,47 @@ internal static partial class HeifTransferFunctions
             case CicpTransferCharacteristics.ItuRBt601_7:
             case CicpTransferCharacteristics.ItuRBt2020_2_10bit:
             case CicpTransferCharacteristics.ItuRBt2020_2_12bit:
-                return ToGammaBt709<TVector, TOperator>(value);
+                return ToGammaBt709<TVector, TOperations>(value);
             case CicpTransferCharacteristics.Gamma2_2:
-                return Power<TVector, TOperator>(TOperator.Min(TOperator.Max(value, zero), one), 1F / 2.2F);
+                return Power<TVector, TOperations>(TOperations.Min(TOperations.Max(value, zero), one), 1F / 2.2F);
             case CicpTransferCharacteristics.Gamma2_8:
-                return Power<TVector, TOperator>(TOperator.Min(TOperator.Max(value, zero), one), 1F / 2.8F);
+                return Power<TVector, TOperations>(TOperations.Min(TOperations.Max(value, zero), one), 1F / 2.8F);
             case CicpTransferCharacteristics.SmpteSt240:
-                return ToGammaSmpte240<TVector, TOperator>(value);
+                return ToGammaSmpte240<TVector, TOperations>(value);
             case CicpTransferCharacteristics.Linear:
-                return TOperator.Min(TOperator.Max(value, zero), one);
+                return TOperations.Min(TOperations.Max(value, zero), one);
             case CicpTransferCharacteristics.Log100:
             {
                 // Clamp inactive lanes to the threshold before Log. ConditionalSelect does not short-circuit,
                 // so this prevents negative input lanes from contaminating the vector operation with NaN values.
-                TVector threshold = TOperator.Create(0.01F);
-                TVector bounded = TOperator.Min(TOperator.Max(value, threshold), one);
-                TVector positive = TOperator.Add(one, TOperator.Divide(TOperator.Log(bounded), TOperator.Create(2F * 2.302585092994046F)));
-                return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, threshold), zero, positive);
+                TVector threshold = TOperations.Create(0.01F);
+                TVector bounded = TOperations.Min(TOperations.Max(value, threshold), one);
+                TVector positive = TOperations.Add(one, TOperations.Divide(TOperations.Log(bounded), TOperations.Create(2F * 2.302585092994046F)));
+                return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, threshold), zero, positive);
             }
 
             case CicpTransferCharacteristics.Log100Sqrt:
             {
-                TVector threshold = TOperator.Create(0.00316227766F);
-                TVector bounded = TOperator.Min(TOperator.Max(value, threshold), one);
-                TVector positive = TOperator.Add(one, TOperator.Divide(TOperator.Log(bounded), TOperator.Create(2.5F * 2.302585092994046F)));
-                return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, threshold), zero, positive);
+                TVector threshold = TOperations.Create(0.00316227766F);
+                TVector bounded = TOperations.Min(TOperations.Max(value, threshold), one);
+                TVector positive = TOperations.Add(one, TOperations.Divide(TOperations.Log(bounded), TOperations.Create(2.5F * 2.302585092994046F)));
+                return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, threshold), zero, positive);
             }
 
             case CicpTransferCharacteristics.Iec61966_2_4:
-                return ToGammaIec61966<TVector, TOperator>(value);
+                return ToGammaIec61966<TVector, TOperations>(value);
             case CicpTransferCharacteristics.ItuRBt1361_0:
-                return ToGammaBt1361<TVector, TOperator>(value);
+                return ToGammaBt1361<TVector, TOperations>(value);
             case CicpTransferCharacteristics.Iec61966_2_1:
-                return ToGammaSrgb<TVector, TOperator>(value);
+                return ToGammaSrgb<TVector, TOperations>(value);
             case CicpTransferCharacteristics.SmpteSt2084:
-                return ToGammaPq<TVector, TOperator>(value);
+                return ToGammaPq<TVector, TOperations>(value);
             case CicpTransferCharacteristics.SmpteSt428_1:
-                return Power<TVector, TOperator>(TOperator.Multiply(TOperator.Create(Smpte428Scale), TOperator.Max(value, zero)), 1F / 2.6F);
+                return Power<TVector, TOperations>(TOperations.Multiply(TOperations.Create(Smpte428Scale), TOperations.Max(value, zero)), 1F / 2.6F);
             case CicpTransferCharacteristics.AribStdB67:
-                return ToGammaHlg<TVector, TOperator>(value);
+                return ToGammaHlg<TVector, TOperations>(value);
             default:
-                return ToGammaBt709<TVector, TOperator>(value);
+                return ToGammaBt709<TVector, TOperations>(value);
         }
     }
 
@@ -201,331 +201,331 @@ internal static partial class HeifTransferFunctions
     /// Applies the inverse BT.709-family opto-electronic transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The linear signal values.</returns>
-    private static TVector ToLinearBt709<TVector, TOperator>(TVector value)
+    private static TVector ToLinearBt709<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector one = TOperator.Create(1F);
-        TVector linear = TOperator.Divide(value, TOperator.Create(4.5F));
-        TVector baseValue = TOperator.Divide(TOperator.Add(value, TOperator.Create(Bt709Alpha - 1F)), TOperator.Create(Bt709Alpha));
-        TVector nonlinear = Power<TVector, TOperator>(TOperator.Max(baseValue, zero), 1F / 0.45F);
-        TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(4.5F * Bt709Beta)), linear, nonlinear);
+        TVector zero = TOperations.Create(0F);
+        TVector one = TOperations.Create(1F);
+        TVector linear = TOperations.Divide(value, TOperations.Create(4.5F));
+        TVector baseValue = TOperations.Divide(TOperations.Add(value, TOperations.Create(Bt709Alpha - 1F)), TOperations.Create(Bt709Alpha));
+        TVector nonlinear = Power<TVector, TOperations>(TOperations.Max(baseValue, zero), 1F / 0.45F);
+        TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(4.5F * Bt709Beta)), linear, nonlinear);
 
         // The comparisons deliberately mirror the scalar ordering. This preserves the H.273 lower and upper
         // saturation rules while allowing all lanes to execute without data-dependent branches.
-        TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded);
+        TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded);
     }
 
     /// <summary>
     /// Applies the BT.709-family opto-electronic transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaBt709<TVector, TOperator>(TVector value)
+    private static TVector ToGammaBt709<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector one = TOperator.Create(1F);
-        TVector linear = TOperator.Multiply(value, TOperator.Create(4.5F));
-        TVector nonlinear = TOperator.Subtract(
-            TOperator.Multiply(TOperator.Create(Bt709Alpha), Power<TVector, TOperator>(TOperator.Max(value, zero), 0.45F)),
-            TOperator.Create(Bt709Alpha - 1F));
-        TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(Bt709Beta)), linear, nonlinear);
-        TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded);
+        TVector zero = TOperations.Create(0F);
+        TVector one = TOperations.Create(1F);
+        TVector linear = TOperations.Multiply(value, TOperations.Create(4.5F));
+        TVector nonlinear = TOperations.Subtract(
+            TOperations.Multiply(TOperations.Create(Bt709Alpha), Power<TVector, TOperations>(TOperations.Max(value, zero), 0.45F)),
+            TOperations.Create(Bt709Alpha - 1F));
+        TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(Bt709Beta)), linear, nonlinear);
+        TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded);
     }
 
     /// <summary>
     /// Applies the inverse SMPTE ST 240 opto-electronic transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The linear signal values.</returns>
-    private static TVector ToLinearSmpte240<TVector, TOperator>(TVector value)
+    private static TVector ToLinearSmpte240<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector one = TOperator.Create(1F);
-        TVector linear = TOperator.Divide(value, TOperator.Create(4F));
-        TVector baseValue = TOperator.Divide(TOperator.Add(value, TOperator.Create(Smpte240Alpha - 1F)), TOperator.Create(Smpte240Alpha));
-        TVector nonlinear = Power<TVector, TOperator>(TOperator.Max(baseValue, zero), 1F / 0.45F);
-        TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(4F * Smpte240Beta)), linear, nonlinear);
-        TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded);
+        TVector zero = TOperations.Create(0F);
+        TVector one = TOperations.Create(1F);
+        TVector linear = TOperations.Divide(value, TOperations.Create(4F));
+        TVector baseValue = TOperations.Divide(TOperations.Add(value, TOperations.Create(Smpte240Alpha - 1F)), TOperations.Create(Smpte240Alpha));
+        TVector nonlinear = Power<TVector, TOperations>(TOperations.Max(baseValue, zero), 1F / 0.45F);
+        TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(4F * Smpte240Beta)), linear, nonlinear);
+        TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded);
     }
 
     /// <summary>
     /// Applies the SMPTE ST 240 opto-electronic transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaSmpte240<TVector, TOperator>(TVector value)
+    private static TVector ToGammaSmpte240<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector one = TOperator.Create(1F);
-        TVector linear = TOperator.Multiply(value, TOperator.Create(4F));
-        TVector nonlinear = TOperator.Subtract(
-            TOperator.Multiply(TOperator.Create(Smpte240Alpha), Power<TVector, TOperator>(TOperator.Max(value, zero), 0.45F)),
-            TOperator.Create(Smpte240Alpha - 1F));
-        TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(Smpte240Beta)), linear, nonlinear);
-        TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded);
+        TVector zero = TOperations.Create(0F);
+        TVector one = TOperations.Create(1F);
+        TVector linear = TOperations.Multiply(value, TOperations.Create(4F));
+        TVector nonlinear = TOperations.Subtract(
+            TOperations.Multiply(TOperations.Create(Smpte240Alpha), Power<TVector, TOperations>(TOperations.Max(value, zero), 0.45F)),
+            TOperations.Create(Smpte240Alpha - 1F));
+        TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(Smpte240Beta)), linear, nonlinear);
+        TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded);
     }
 
     /// <summary>
     /// Applies the inverse extended IEC 61966-2-4 transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The linear signal values.</returns>
-    private static TVector ToLinearIec61966<TVector, TOperator>(TVector value)
+    private static TVector ToLinearIec61966<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector negativeBase = TOperator.Divide(TOperator.Subtract(value, TOperator.Create(Bt709Alpha - 1F)), TOperator.Create(-Bt709Alpha));
-        TVector negative = TOperator.Negate(Power<TVector, TOperator>(TOperator.Max(negativeBase, TOperator.Create(0F)), 1F / 0.45F));
-        TVector linear = TOperator.Divide(value, TOperator.Create(4.5F));
-        TVector positiveBase = TOperator.Divide(TOperator.Add(value, TOperator.Create(Bt709Alpha - 1F)), TOperator.Create(Bt709Alpha));
-        TVector positive = Power<TVector, TOperator>(TOperator.Max(positiveBase, TOperator.Create(0F)), 1F / 0.45F);
-        TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(4.5F * Bt709Beta)), linear, positive);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-4.5F * Bt709Beta)), negative, centerOrPositive);
+        TVector negativeBase = TOperations.Divide(TOperations.Subtract(value, TOperations.Create(Bt709Alpha - 1F)), TOperations.Create(-Bt709Alpha));
+        TVector negative = TOperations.Negate(Power<TVector, TOperations>(TOperations.Max(negativeBase, TOperations.Create(0F)), 1F / 0.45F));
+        TVector linear = TOperations.Divide(value, TOperations.Create(4.5F));
+        TVector positiveBase = TOperations.Divide(TOperations.Add(value, TOperations.Create(Bt709Alpha - 1F)), TOperations.Create(Bt709Alpha));
+        TVector positive = Power<TVector, TOperations>(TOperations.Max(positiveBase, TOperations.Create(0F)), 1F / 0.45F);
+        TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(4.5F * Bt709Beta)), linear, positive);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-4.5F * Bt709Beta)), negative, centerOrPositive);
     }
 
     /// <summary>
     /// Applies the extended IEC 61966-2-4 transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaIec61966<TVector, TOperator>(TVector value)
+    private static TVector ToGammaIec61966<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector negative = TOperator.Add(
-            TOperator.Negate(TOperator.Multiply(
-                TOperator.Create(Bt709Alpha),
-                Power<TVector, TOperator>(TOperator.Max(TOperator.Negate(value), zero), 0.45F))),
-            TOperator.Create(Bt709Alpha - 1F));
-
-        TVector linear = TOperator.Multiply(value, TOperator.Create(4.5F));
-        TVector positive = TOperator.Subtract(
-            TOperator.Multiply(TOperator.Create(Bt709Alpha), Power<TVector, TOperator>(TOperator.Max(value, zero), 0.45F)),
-            TOperator.Create(Bt709Alpha - 1F));
-        TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(Bt709Beta)), linear, positive);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-Bt709Beta)), negative, centerOrPositive);
+        TVector zero = TOperations.Create(0F);
+        TVector negative = TOperations.Add(
+            TOperations.Negate(TOperations.Multiply(
+                TOperations.Create(Bt709Alpha),
+                Power<TVector, TOperations>(TOperations.Max(TOperations.Negate(value), zero), 0.45F))),
+            TOperations.Create(Bt709Alpha - 1F));
+
+        TVector linear = TOperations.Multiply(value, TOperations.Create(4.5F));
+        TVector positive = TOperations.Subtract(
+            TOperations.Multiply(TOperations.Create(Bt709Alpha), Power<TVector, TOperations>(TOperations.Max(value, zero), 0.45F)),
+            TOperations.Create(Bt709Alpha - 1F));
+        TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(Bt709Beta)), linear, positive);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-Bt709Beta)), negative, centerOrPositive);
     }
 
     /// <summary>
     /// Applies the inverse extended BT.1361 transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The linear signal values.</returns>
-    private static TVector ToLinearBt1361<TVector, TOperator>(TVector value)
+    private static TVector ToLinearBt1361<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector negativeBase = TOperator.Divide(TOperator.Subtract(value, TOperator.Create(0.02482420670236F)), TOperator.Create(-0.27482420670236F));
-        TVector negative = TOperator.Divide(Power<TVector, TOperator>(TOperator.Max(negativeBase, zero), 1F / 0.45F), TOperator.Create(-4F));
-        TVector negativeOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, zero), negative, ToLinearBt709<TVector, TOperator>(value));
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-0.25F)), TOperator.Create(-0.25F), negativeOrPositive);
+        TVector zero = TOperations.Create(0F);
+        TVector negativeBase = TOperations.Divide(TOperations.Subtract(value, TOperations.Create(0.02482420670236F)), TOperations.Create(-0.27482420670236F));
+        TVector negative = TOperations.Divide(Power<TVector, TOperations>(TOperations.Max(negativeBase, zero), 1F / 0.45F), TOperations.Create(-4F));
+        TVector negativeOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, zero), negative, ToLinearBt709<TVector, TOperations>(value));
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-0.25F)), TOperations.Create(-0.25F), negativeOrPositive);
     }
 
     /// <summary>
     /// Applies the extended BT.1361 transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaBt1361<TVector, TOperator>(TVector value)
+    private static TVector ToGammaBt1361<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector negativePower = Power<TVector, TOperator>(TOperator.Max(TOperator.Multiply(TOperator.Create(-4F), value), zero), 0.45F);
-        TVector negative = TOperator.Add(TOperator.Multiply(TOperator.Create(-0.27482420670236F), negativePower), TOperator.Create(0.02482420670236F));
-        TVector negativeOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, zero), negative, ToGammaBt709<TVector, TOperator>(value));
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-0.25F)), TOperator.Create(-0.25F), negativeOrPositive);
+        TVector zero = TOperations.Create(0F);
+        TVector negativePower = Power<TVector, TOperations>(TOperations.Max(TOperations.Multiply(TOperations.Create(-4F), value), zero), 0.45F);
+        TVector negative = TOperations.Add(TOperations.Multiply(TOperations.Create(-0.27482420670236F), negativePower), TOperations.Create(0.02482420670236F));
+        TVector negativeOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, zero), negative, ToGammaBt709<TVector, TOperations>(value));
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-0.25F)), TOperations.Create(-0.25F), negativeOrPositive);
     }
 
     /// <summary>
     /// Applies the inverse extended IEC 61966-2-1 transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The linear signal values.</returns>
-    private static TVector ToLinearSrgb<TVector, TOperator>(TVector value)
+    private static TVector ToLinearSrgb<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector negativeBase = TOperator.Divide(TOperator.Subtract(value, TOperator.Create(SrgbAlpha - 1F)), TOperator.Create(-SrgbAlpha));
-        TVector negative = TOperator.Negate(Power<TVector, TOperator>(TOperator.Max(negativeBase, zero), 2.4F));
-        TVector linear = TOperator.Divide(value, TOperator.Create(12.92F));
-        TVector positiveBase = TOperator.Divide(TOperator.Add(value, TOperator.Create(SrgbAlpha - 1F)), TOperator.Create(SrgbAlpha));
-        TVector positive = Power<TVector, TOperator>(TOperator.Max(positiveBase, zero), 2.4F);
-        TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(12.92F * SrgbBeta)), linear, positive);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-12.92F * SrgbBeta)), negative, centerOrPositive);
+        TVector zero = TOperations.Create(0F);
+        TVector negativeBase = TOperations.Divide(TOperations.Subtract(value, TOperations.Create(SrgbAlpha - 1F)), TOperations.Create(-SrgbAlpha));
+        TVector negative = TOperations.Negate(Power<TVector, TOperations>(TOperations.Max(negativeBase, zero), 2.4F));
+        TVector linear = TOperations.Divide(value, TOperations.Create(12.92F));
+        TVector positiveBase = TOperations.Divide(TOperations.Add(value, TOperations.Create(SrgbAlpha - 1F)), TOperations.Create(SrgbAlpha));
+        TVector positive = Power<TVector, TOperations>(TOperations.Max(positiveBase, zero), 2.4F);
+        TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(12.92F * SrgbBeta)), linear, positive);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-12.92F * SrgbBeta)), negative, centerOrPositive);
     }
 
     /// <summary>
     /// Applies the extended IEC 61966-2-1 transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaSrgb<TVector, TOperator>(TVector value)
+    private static TVector ToGammaSrgb<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector negative = TOperator.Add(
-            TOperator.Negate(TOperator.Multiply(
-                TOperator.Create(SrgbAlpha),
-                Power<TVector, TOperator>(TOperator.Max(TOperator.Negate(value), zero), 1F / 2.4F))),
-            TOperator.Create(SrgbAlpha - 1F));
-
-        TVector linear = TOperator.Multiply(value, TOperator.Create(12.92F));
-        TVector positive = TOperator.Subtract(
-            TOperator.Multiply(TOperator.Create(SrgbAlpha), Power<TVector, TOperator>(TOperator.Max(value, zero), 1F / 2.4F)),
-            TOperator.Create(SrgbAlpha - 1F));
-        TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(SrgbBeta)), linear, positive);
-        return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-SrgbBeta)), negative, centerOrPositive);
+        TVector zero = TOperations.Create(0F);
+        TVector negative = TOperations.Add(
+            TOperations.Negate(TOperations.Multiply(
+                TOperations.Create(SrgbAlpha),
+                Power<TVector, TOperations>(TOperations.Max(TOperations.Negate(value), zero), 1F / 2.4F))),
+            TOperations.Create(SrgbAlpha - 1F));
+
+        TVector linear = TOperations.Multiply(value, TOperations.Create(12.92F));
+        TVector positive = TOperations.Subtract(
+            TOperations.Multiply(TOperations.Create(SrgbAlpha), Power<TVector, TOperations>(TOperations.Max(value, zero), 1F / 2.4F)),
+            TOperations.Create(SrgbAlpha - 1F));
+        TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(SrgbBeta)), linear, positive);
+        return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-SrgbBeta)), negative, centerOrPositive);
     }
 
     /// <summary>
     /// Applies the inverse SMPTE ST 2084 perceptual-quantizer transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The normalized linear signal values.</returns>
-    private static TVector ToLinearPq<TVector, TOperator>(TVector value)
+    private static TVector ToLinearPq<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector nonlinearPower = Power<TVector, TOperator>(TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F)), 1F / PqM);
-        TVector numerator = TOperator.Max(TOperator.Subtract(nonlinearPower, TOperator.Create(PqC1)), zero);
-        TVector denominator = TOperator.Subtract(TOperator.Create(PqC2), TOperator.Multiply(TOperator.Create(PqC3), nonlinearPower));
-        TVector positive = TOperator.Min(Power<TVector, TOperator>(TOperator.Divide(numerator, denominator), 1F / PqN), TOperator.Create(1F));
-        return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive);
+        TVector zero = TOperations.Create(0F);
+        TVector nonlinearPower = Power<TVector, TOperations>(TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F)), 1F / PqM);
+        TVector numerator = TOperations.Max(TOperations.Subtract(nonlinearPower, TOperations.Create(PqC1)), zero);
+        TVector denominator = TOperations.Subtract(TOperations.Create(PqC2), TOperations.Multiply(TOperations.Create(PqC3), nonlinearPower));
+        TVector positive = TOperations.Min(Power<TVector, TOperations>(TOperations.Divide(numerator, denominator), 1F / PqN), TOperations.Create(1F));
+        return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive);
     }
 
     /// <summary>
     /// Applies the SMPTE ST 2084 perceptual-quantizer transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The normalized linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaPq<TVector, TOperator>(TVector value)
+    private static TVector ToGammaPq<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector linearPower = Power<TVector, TOperator>(TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F)), PqN);
-        TVector numerator = TOperator.Add(TOperator.Create(PqC1), TOperator.Multiply(TOperator.Create(PqC2), linearPower));
-        TVector denominator = TOperator.Add(TOperator.Create(1F), TOperator.Multiply(TOperator.Create(PqC3), linearPower));
-        TVector positive = TOperator.Min(Power<TVector, TOperator>(TOperator.Divide(numerator, denominator), PqM), TOperator.Create(1F));
-        return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive);
+        TVector zero = TOperations.Create(0F);
+        TVector linearPower = Power<TVector, TOperations>(TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F)), PqN);
+        TVector numerator = TOperations.Add(TOperations.Create(PqC1), TOperations.Multiply(TOperations.Create(PqC2), linearPower));
+        TVector denominator = TOperations.Add(TOperations.Create(1F), TOperations.Multiply(TOperations.Create(PqC3), linearPower));
+        TVector positive = TOperations.Min(Power<TVector, TOperations>(TOperations.Divide(numerator, denominator), PqM), TOperations.Create(1F));
+        return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive);
     }
 
     /// <summary>
     /// Applies the inverse HLG opto-electronic transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonlinear signal values.</param>
     /// <returns>The normalized linear signal values.</returns>
-    private static TVector ToLinearHlg<TVector, TOperator>(TVector value)
+    private static TVector ToLinearHlg<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector bounded = TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F));
-        TVector linear = TOperator.Divide(TOperator.Multiply(bounded, bounded), TOperator.Create(3F));
-        TVector exponent = TOperator.Divide(TOperator.Subtract(bounded, TOperator.Create(HlgC)), TOperator.Create(HlgA));
-        TVector logarithmic = TOperator.Divide(TOperator.Add(TOperator.Exp(exponent), TOperator.Create(HlgB)), TOperator.Create(12F));
-        TVector positive = TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, TOperator.Create(0.5F)), linear, logarithmic);
-        return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive);
+        TVector zero = TOperations.Create(0F);
+        TVector bounded = TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F));
+        TVector linear = TOperations.Divide(TOperations.Multiply(bounded, bounded), TOperations.Create(3F));
+        TVector exponent = TOperations.Divide(TOperations.Subtract(bounded, TOperations.Create(HlgC)), TOperations.Create(HlgA));
+        TVector logarithmic = TOperations.Divide(TOperations.Add(TOperations.Exp(exponent), TOperations.Create(HlgB)), TOperations.Create(12F));
+        TVector positive = TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, TOperations.Create(0.5F)), linear, logarithmic);
+        return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive);
     }
 
     /// <summary>
     /// Applies the HLG opto-electronic transfer function to a SIMD vector.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The normalized linear signal values.</param>
     /// <returns>The nonlinear signal values.</returns>
-    private static TVector ToGammaHlg<TVector, TOperator>(TVector value)
+    private static TVector ToGammaHlg<TVector, TOperations>(TVector value)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
-        TVector zero = TOperator.Create(0F);
-        TVector bounded = TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F));
-        TVector linear = TOperator.Sqrt(TOperator.Multiply(TOperator.Create(3F), bounded));
+        TVector zero = TOperations.Create(0F);
+        TVector bounded = TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F));
+        TVector linear = TOperations.Sqrt(TOperations.Multiply(TOperations.Create(3F), bounded));
 
         // Clamp the logarithm input for inactive lanes. SIMD conditional selection evaluates both branches,
         // while the scalar definition evaluates Log only above the 1/12 transition.
-        TVector logarithmInput = TOperator.Max(
-            TOperator.Subtract(TOperator.Multiply(TOperator.Create(12F), bounded), TOperator.Create(HlgB)),
-            TOperator.Create(float.Epsilon));
-        TVector logarithmic = TOperator.Add(TOperator.Multiply(TOperator.Create(HlgA), TOperator.Log(logarithmInput)), TOperator.Create(HlgC));
-        TVector positive = TOperator.ConditionalSelect(TOperator.LessThanOrEqual(bounded, TOperator.Create(1F / 12F)), linear, logarithmic);
-        return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive);
+        TVector logarithmInput = TOperations.Max(
+            TOperations.Subtract(TOperations.Multiply(TOperations.Create(12F), bounded), TOperations.Create(HlgB)),
+            TOperations.Create(float.Epsilon));
+        TVector logarithmic = TOperations.Add(TOperations.Multiply(TOperations.Create(HlgA), TOperations.Log(logarithmInput)), TOperations.Create(HlgC));
+        TVector positive = TOperations.ConditionalSelect(TOperations.LessThanOrEqual(bounded, TOperations.Create(1F / 12F)), linear, logarithmic);
+        return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive);
     }
 
     /// <summary>
     /// Raises nonnegative SIMD values to a scalar exponent.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    /// <typeparam name="TOperator">The operations for the SIMD vector type.</typeparam>
+    /// <typeparam name="TOperations">The operations for the SIMD vector type.</typeparam>
     /// <param name="value">The nonnegative base values.</param>
     /// <param name="exponent">The exponent applied to every lane.</param>
     /// <returns>The exponentiated values.</returns>
-    private static TVector Power<TVector, TOperator>(TVector value, float exponent)
+    private static TVector Power<TVector, TOperations>(TVector value, float exponent)
         where TVector : struct
-        where TOperator : struct, ITransferVectorOperator<TVector>
+        where TOperations : struct, ITransferVectorOperations<TVector>
     {
         // System.Numerics.Tensors does not currently vectorize Pow. Expressing positive powers as Exp(Log(x) * y)
         // uses the .NET 10 cross-platform vector math kernels and keeps all transfer-function lanes in SIMD.
-        return TOperator.Exp(TOperator.Multiply(TOperator.Log(value), TOperator.Create(exponent)));
+        return TOperations.Exp(TOperations.Multiply(TOperations.Log(value), TOperations.Create(exponent)));
     }
 }
 
 /// <summary>
-/// Contains the stateless vector-width operators used by the shared H.273 transfer-function formulas.
+/// Contains the vector-width operations used by the shared H.273 transfer-function formulas.
 /// </summary>
-internal static class HeifTransferVectorOperators
+internal static class HeifTransferVectorOperations
 {
     /// <summary>
     /// Defines the lane-wise operations required by the shared H.273 SIMD formulas.
     /// </summary>
     /// <typeparam name="TVector">The SIMD vector type.</typeparam>
-    public interface ITransferVectorOperator<TVector>
+    public interface ITransferVectorOperations<TVector>
         where TVector : struct
     {
         /// <summary>
@@ -649,7 +649,7 @@ internal static class HeifTransferVectorOperators
     /// <summary>
     /// Maps the shared transfer-function formulas to 128-bit vector operations.
     /// </summary>
-    public readonly struct Vector128Operator : ITransferVectorOperator<Vector128<float>>
+    public readonly struct Vector128Operations : ITransferVectorOperations<Vector128<float>>
     {
         /// <inheritdoc/>
         public static Vector128<float> Create(float value) => Vector128.Create(value);
@@ -702,7 +702,7 @@ internal static class HeifTransferVectorOperators
     /// <summary>
     /// Maps the shared transfer-function formulas to 256-bit vector operations.
     /// </summary>
-    public readonly struct Vector256Operator : ITransferVectorOperator<Vector256<float>>
+    public readonly struct Vector256Operations : ITransferVectorOperations<Vector256<float>>
     {
         /// <inheritdoc/>
         public static Vector256<float> Create(float value) => Vector256.Create(value);
@@ -755,7 +755,7 @@ internal static class HeifTransferVectorOperators
     /// <summary>
     /// Maps the shared transfer-function formulas to 512-bit vector operations.
     /// </summary>
-    public readonly struct Vector512Operator : ITransferVectorOperator<Vector512<float>>
+    public readonly struct Vector512Operations : ITransferVectorOperations<Vector512<float>>
     {
         /// <inheritdoc/>
         public static Vector512<float> Create(float value) => Vector512.Create(value);

src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.HorizontalEdgeOperator.cs

@@ -0,0 +1,64 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+internal static partial class HevcDeblockingFilter
+{
+    /// <summary>
+    /// Accesses four columns across a horizontal edge.
+    /// </summary>
+    private readonly struct HorizontalEdgeOperator : IEdgeOperator
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count)
+        {
+            ref ushort source = ref picture.GetRowSpan(plane, y + distance)[x];
+            if (count == 2)
+            {
+                // The packed load used by full-width segments would read two samples beyond a subsampled edge.
+                return Vector128.Create((int)source, Unsafe.Add(ref source, 1), 0, 0);
+            }
+
+            Vector64<ushort> packed = Unsafe.As<ushort, Vector64<ushort>>(ref source);
+            return Vector128.WidenLower(Vector128.Create(packed, Vector64<ushort>.Zero)).AsInt32();
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreVector(
+            HevcPictureBuffer picture,
+            HevcPlane plane,
+            int x,
+            int y,
+            int distance,
+            Vector128<int> value,
+            int count)
+        {
+            ref ushort destination = ref picture.GetRowSpan(plane, y + distance)[x];
+            if (count == 4)
+            {
+                Vector64<ushort> packed = Vector128.Narrow(value, Vector128<int>.Zero).AsUInt16().GetLower();
+                Unsafe.As<ushort, Vector64<ushort>>(ref destination) = packed;
+                return;
+            }
+
+            destination = (ushort)value.GetElement(0);
+            Unsafe.Add(ref destination, 1) = (ushort)value.GetElement(1);
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index)
+            => picture.GetRowSpan(plane, y + distance)[x + index];
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value)
+            => picture.GetRowSpan(plane, y + distance)[x + index] = (ushort)value;
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.Operator.cs

@@ -0,0 +1,70 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+internal static partial class HevcDeblockingFilter
+{
+    /// <summary>
+    /// Defines orientation-specific access to the four samples running along one deblocking edge segment.
+    /// </summary>
+    private interface IEdgeOperator
+    {
+        /// <summary>
+        /// Loads samples at one signed distance across the edge.
+        /// </summary>
+        /// <param name="picture">The reconstructed picture.</param>
+        /// <param name="plane">The component plane.</param>
+        /// <param name="x">The first Q-side sample X coordinate.</param>
+        /// <param name="y">The first Q-side sample Y coordinate.</param>
+        /// <param name="distance">The signed sample distance across the edge.</param>
+        /// <param name="count">The number of valid low lanes to load.</param>
+        /// <returns>The widened samples ordered along the edge.</returns>
+        public static abstract Vector128<int> LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count);
+
+        /// <summary>
+        /// Stores four samples at one signed distance across the edge.
+        /// </summary>
+        /// <param name="picture">The reconstructed picture.</param>
+        /// <param name="plane">The component plane.</param>
+        /// <param name="x">The first Q-side sample X coordinate.</param>
+        /// <param name="y">The first Q-side sample Y coordinate.</param>
+        /// <param name="distance">The signed sample distance across the edge.</param>
+        /// <param name="value">The four widened samples ordered along the edge.</param>
+        /// <param name="count">The number of low lanes to store.</param>
+        public static abstract void StoreVector(
+            HevcPictureBuffer picture,
+            HevcPlane plane,
+            int x,
+            int y,
+            int distance,
+            Vector128<int> value,
+            int count);
+
+        /// <summary>
+        /// Loads one scalar sample at a signed distance across and an offset along the edge.
+        /// </summary>
+        /// <param name="picture">The reconstructed picture.</param>
+        /// <param name="plane">The component plane.</param>
+        /// <param name="x">The first Q-side sample X coordinate.</param>
+        /// <param name="y">The first Q-side sample Y coordinate.</param>
+        /// <param name="distance">The signed sample distance across the edge.</param>
+        /// <param name="index">The sample offset along the edge.</param>
+        /// <returns>The selected sample.</returns>
+        public static abstract int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index);
+
+        /// <summary>
+        /// Stores one scalar sample at a signed distance across and an offset along the edge.
+        /// </summary>
+        /// <param name="picture">The reconstructed picture.</param>
+        /// <param name="plane">The component plane.</param>
+        /// <param name="x">The first Q-side sample X coordinate.</param>
+        /// <param name="y">The first Q-side sample Y coordinate.</param>
+        /// <param name="distance">The signed sample distance across the edge.</param>
+        /// <param name="index">The sample offset along the edge.</param>
+        /// <param name="value">The filtered sample.</param>
+        public static abstract void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value);
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.VerticalEdgeOperator.cs

@@ -0,0 +1,65 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+internal static partial class HevcDeblockingFilter
+{
+    /// <summary>
+    /// Accesses four rows across a vertical edge.
+    /// </summary>
+    private readonly struct VerticalEdgeOperator : IEdgeOperator
+    {
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Vector128<int> LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count)
+        {
+            if (count == 4)
+            {
+                return Vector128.Create(
+                    (int)picture.GetRowSpan(plane, y)[x + distance],
+                    picture.GetRowSpan(plane, y + 1)[x + distance],
+                    picture.GetRowSpan(plane, y + 2)[x + distance],
+                    picture.GetRowSpan(plane, y + 3)[x + distance]);
+            }
+
+            // Subsampled chroma edges contain two samples. Zeroing the unused lanes keeps the vector path within
+            // the plane while allowing the shared kernel to operate on both valid samples in one instruction stream.
+            return Vector128.Create(
+                (int)picture.GetRowSpan(plane, y)[x + distance],
+                picture.GetRowSpan(plane, y + 1)[x + distance],
+                0,
+                0);
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreVector(
+            HevcPictureBuffer picture,
+            HevcPlane plane,
+            int x,
+            int y,
+            int distance,
+            Vector128<int> value,
+            int count)
+        {
+            for (int index = 0; index < count; index++)
+            {
+                picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value.GetElement(index);
+            }
+        }
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index)
+            => picture.GetRowSpan(plane, y + index)[x + distance];
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value)
+            => picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value;
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs

@@ -15,69 +15,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
 /// Loads with fewer than four valid positions populate only the low lanes, which the matching store writes without
 /// touching samples beyond the picture boundary.
 /// </remarks>
-internal static class HevcDeblockingFilter
+internal static partial class HevcDeblockingFilter
 {
-    /// <summary>
-    /// Defines orientation-specific access to the four samples running along one deblocking edge segment.
-    /// </summary>
-    private interface IEdgeOperator
-    {
-        /// <summary>
-        /// Loads samples at one signed distance across the edge.
-        /// </summary>
-        /// <param name="picture">The reconstructed picture.</param>
-        /// <param name="plane">The component plane.</param>
-        /// <param name="x">The first Q-side sample X coordinate.</param>
-        /// <param name="y">The first Q-side sample Y coordinate.</param>
-        /// <param name="distance">The signed sample distance across the edge.</param>
-        /// <param name="count">The number of valid low lanes to load.</param>
-        /// <returns>The widened samples ordered along the edge.</returns>
-        public static abstract Vector128<int> LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count);
-
-        /// <summary>
-        /// Stores four samples at one signed distance across the edge.
-        /// </summary>
-        /// <param name="picture">The reconstructed picture.</param>
-        /// <param name="plane">The component plane.</param>
-        /// <param name="x">The first Q-side sample X coordinate.</param>
-        /// <param name="y">The first Q-side sample Y coordinate.</param>
-        /// <param name="distance">The signed sample distance across the edge.</param>
-        /// <param name="value">The four widened samples ordered along the edge.</param>
-        /// <param name="count">The number of low lanes to store.</param>
-        public static abstract void StoreVector(
-            HevcPictureBuffer picture,
-            HevcPlane plane,
-            int x,
-            int y,
-            int distance,
-            Vector128<int> value,
-            int count);
-
-        /// <summary>
-        /// Loads one scalar sample at a signed distance across and an offset along the edge.
-        /// </summary>
-        /// <param name="picture">The reconstructed picture.</param>
-        /// <param name="plane">The component plane.</param>
-        /// <param name="x">The first Q-side sample X coordinate.</param>
-        /// <param name="y">The first Q-side sample Y coordinate.</param>
-        /// <param name="distance">The signed sample distance across the edge.</param>
-        /// <param name="index">The sample offset along the edge.</param>
-        /// <returns>The selected sample.</returns>
-        public static abstract int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index);
-
-        /// <summary>
-        /// Stores one scalar sample at a signed distance across and an offset along the edge.
-        /// </summary>
-        /// <param name="picture">The reconstructed picture.</param>
-        /// <param name="plane">The component plane.</param>
-        /// <param name="x">The first Q-side sample X coordinate.</param>
-        /// <param name="y">The first Q-side sample Y coordinate.</param>
-        /// <param name="distance">The signed sample distance across the edge.</param>
-        /// <param name="index">The sample offset along the edge.</param>
-        /// <param name="value">The filtered sample.</param>
-        public static abstract void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value);
-    }
-
     /// <summary>
     /// Filters four rows crossing one vertical luma boundary.
     /// </summary>
@@ -528,113 +467,4 @@ internal static class HevcDeblockingFilter
             && discontinuity < (beta >> 2)
             && Math.Abs(p0 - q0) < strongThreshold;
     }
-
-    /// <summary>
-    /// Accesses four rows across a vertical edge.
-    /// </summary>
-    private readonly struct VerticalEdgeOperator : IEdgeOperator
-    {
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static Vector128<int> LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count)
-        {
-            if (count == 4)
-            {
-                return Vector128.Create(
-                    (int)picture.GetRowSpan(plane, y)[x + distance],
-                    picture.GetRowSpan(plane, y + 1)[x + distance],
-                    picture.GetRowSpan(plane, y + 2)[x + distance],
-                    picture.GetRowSpan(plane, y + 3)[x + distance]);
-            }
-
-            // Subsampled chroma edges contain two samples. Zeroing the unused lanes keeps the vector path within
-            // the plane while allowing the shared kernel to operate on both valid samples in one instruction stream.
-            return Vector128.Create(
-                (int)picture.GetRowSpan(plane, y)[x + distance],
-                picture.GetRowSpan(plane, y + 1)[x + distance],
-                0,
-                0);
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreVector(
-            HevcPictureBuffer picture,
-            HevcPlane plane,
-            int x,
-            int y,
-            int distance,
-            Vector128<int> value,
-            int count)
-        {
-            for (int index = 0; index < count; index++)
-            {
-                picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value.GetElement(index);
-            }
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index)
-            => picture.GetRowSpan(plane, y + index)[x + distance];
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value)
-            => picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value;
-    }
-
-    /// <summary>
-    /// Accesses four columns across a horizontal edge.
-    /// </summary>
-    private readonly struct HorizontalEdgeOperator : IEdgeOperator
-    {
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static Vector128<int> LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count)
-        {
-            ref ushort source = ref picture.GetRowSpan(plane, y + distance)[x];
-            if (count == 2)
-            {
-                // The packed load used by full-width segments would read two samples beyond a subsampled edge.
-                return Vector128.Create((int)source, Unsafe.Add(ref source, 1), 0, 0);
-            }
-
-            Vector64<ushort> packed = Unsafe.As<ushort, Vector64<ushort>>(ref source);
-            return Vector128.WidenLower(Vector128.Create(packed, Vector64<ushort>.Zero)).AsInt32();
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreVector(
-            HevcPictureBuffer picture,
-            HevcPlane plane,
-            int x,
-            int y,
-            int distance,
-            Vector128<int> value,
-            int count)
-        {
-            ref ushort destination = ref picture.GetRowSpan(plane, y + distance)[x];
-            if (count == 4)
-            {
-                Vector64<ushort> packed = Vector128.Narrow(value, Vector128<int>.Zero).AsUInt16().GetLower();
-                Unsafe.As<ushort, Vector64<ushort>>(ref destination) = packed;
-                return;
-            }
-
-            destination = (ushort)value.GetElement(0);
-            Unsafe.Add(ref destination, 1) = (ushort)value.GetElement(1);
-        }
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index)
-            => picture.GetRowSpan(plane, y + distance)[x + index];
-
-        /// <inheritdoc/>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value)
-            => picture.GetRowSpan(plane, y + distance)[x + index] = (ushort)value;
-    }
 }

src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.AngularOperator.cs

@@ -0,0 +1,346 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines angular intra-prediction arithmetic.
+/// </content>
+internal static partial class HevcIntraPredictor
+{
+    /// <summary>
+    /// Implements the thirty-three directional intra-prediction modes.
+    /// </summary>
+    private readonly struct AngularOperator : IHevcIntraPredictionOperator
+    {
+        /// <inheritdoc/>
+        public static void Predict(
+            ReadOnlySpan<ushort> top,
+            ReadOnlySpan<ushort> left,
+            Span<ushort> destination,
+            int destinationStride,
+            int size,
+            int mode,
+            int bitDepth,
+            bool filterPredictionEdges,
+            Span<ushort> scratch)
+        {
+            if (mode == VerticalMode)
+            {
+                PredictVertical(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges);
+                return;
+            }
+
+            if (mode == HorizontalMode)
+            {
+                PredictHorizontal(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges);
+                return;
+            }
+
+            bool vertical = mode >= FirstVerticalMode;
+            int angleMode = vertical ? mode - VerticalMode : HorizontalMode - mode;
+            int absoluteAngleMode = Math.Abs(angleMode);
+            int angle = PredictionAngles[absoluteAngleMode] * Math.Sign(angleMode);
+            ReadOnlySpan<ushort> main = vertical ? top : left;
+            ReadOnlySpan<ushort> side = vertical ? left : top;
+            Span<ushort> temporaryBlock = scratch[..(size * size)];
+            Span<ushort> extendedReference = scratch.Slice(size * size, (4 * size) + 1);
+            int mainOrigin = 0;
+
+            if (angle < 0)
+            {
+                mainOrigin = size * 2;
+                main[..(size + 1)].CopyTo(extendedReference[mainOrigin..]);
+                int inverseAngle = InversePredictionAngles[absoluteAngleMode];
+                int inverseAngleSum = 128;
+                int minimumIndex = (size * angle) >> 5;
+                for (int index = -1; index > minimumIndex; index--)
+                {
+                    inverseAngleSum += inverseAngle;
+                    extendedReference[mainOrigin + index] = side[inverseAngleSum >> 8];
+                }
+
+                main = extendedReference;
+            }
+
+            Span<ushort> prediction = vertical ? destination : temporaryBlock;
+            int predictionStride = vertical ? destinationStride : size;
+            PredictAngularRows(main, mainOrigin, prediction, predictionStride, size, angle);
+            if (!vertical)
+            {
+                TransposeBlock(temporaryBlock, destination, destinationStride, size);
+            }
+        }
+
+        /// <summary>
+        /// Copies the top reference into every row and optionally filters the first column.
+        /// </summary>
+        /// <param name="top">The top reference samples.</param>
+        /// <param name="left">The left reference samples.</param>
+        /// <param name="destination">The destination block origin.</param>
+        /// <param name="destinationStride">The destination row stride.</param>
+        /// <param name="size">The square block side.</param>
+        /// <param name="bitDepth">The reconstructed component precision.</param>
+        /// <param name="filterPredictionEdges">Whether the vertical luma edge filter applies.</param>
+        private static void PredictVertical(
+            ReadOnlySpan<ushort> top,
+            ReadOnlySpan<ushort> left,
+            Span<ushort> destination,
+            int destinationStride,
+            int size,
+            int bitDepth,
+            bool filterPredictionEdges)
+        {
+            ReadOnlySpan<ushort> row = top.Slice(1, size);
+            int maximum = (1 << bitDepth) - 1;
+            for (int y = 0; y < size; y++)
+            {
+                row.CopyTo(destination.Slice(y * destinationStride, size));
+                if (filterPredictionEdges)
+                {
+                    int sample = destination[y * destinationStride] + ((left[y + 1] - left[0]) >> 1);
+                    destination[y * destinationStride] = (ushort)Math.Clamp(sample, 0, maximum);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Fills each row from its left reference and optionally filters the first row.
+        /// </summary>
+        /// <param name="top">The top reference samples.</param>
+        /// <param name="left">The left reference samples.</param>
+        /// <param name="destination">The destination block origin.</param>
+        /// <param name="destinationStride">The destination row stride.</param>
+        /// <param name="size">The square block side.</param>
+        /// <param name="bitDepth">The reconstructed component precision.</param>
+        /// <param name="filterPredictionEdges">Whether the horizontal luma edge filter applies.</param>
+        private static void PredictHorizontal(
+            ReadOnlySpan<ushort> top,
+            ReadOnlySpan<ushort> left,
+            Span<ushort> destination,
+            int destinationStride,
+            int size,
+            int bitDepth,
+            bool filterPredictionEdges)
+        {
+            for (int y = 0; y < size; y++)
+            {
+                destination.Slice(y * destinationStride, size).Fill(left[y + 1]);
+            }
+
+            if (!filterPredictionEdges)
+            {
+                return;
+            }
+
+            int maximum = (1 << bitDepth) - 1;
+            for (int x = 0; x < size; x++)
+            {
+                int sample = destination[x] + ((top[x + 1] - top[0]) >> 1);
+                destination[x] = (ushort)Math.Clamp(sample, 0, maximum);
+            }
+        }
+
+        /// <summary>
+        /// Generates a vertical-oriented angular block using contiguous SIMD interpolation within each row.
+        /// </summary>
+        /// <param name="main">The main reference beginning at logical index zero.</param>
+        /// <param name="mainOrigin">The span index corresponding to logical reference index zero.</param>
+        /// <param name="destination">The contiguous destination or transposition scratch block.</param>
+        /// <param name="destinationStride">The destination row stride.</param>
+        /// <param name="size">The square block side.</param>
+        /// <param name="angle">The signed prediction displacement in thirty-second-sample units.</param>
+        private static void PredictAngularRows(
+            ReadOnlySpan<ushort> main,
+            int mainOrigin,
+            Span<ushort> destination,
+            int destinationStride,
+            int size,
+            int angle)
+        {
+            for (int y = 0, deltaPosition = angle; y < size; y++, deltaPosition += angle)
+            {
+                int deltaInteger = deltaPosition >> 5;
+                int deltaFraction = deltaPosition & 31;
+                int sourceOffset = mainOrigin + deltaInteger + 1;
+                Span<ushort> row = destination.Slice(y * destinationStride, size);
+                if (deltaFraction == 0)
+                {
+                    main.Slice(sourceOffset, size).CopyTo(row);
+                }
+                else
+                {
+                    InterpolateAngularRow(main[sourceOffset..], row, deltaFraction);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Interpolates one angular prediction row between consecutive main-reference samples.
+        /// </summary>
+        /// <param name="source">The first main-reference sample for the row.</param>
+        /// <param name="destination">The destination prediction row.</param>
+        /// <param name="fraction">The right-hand weight with a denominator of thirty-two.</param>
+        private static void InterpolateAngularRow(ReadOnlySpan<ushort> source, Span<ushort> destination, int fraction)
+        {
+            ref ushort sourceBase = ref MemoryMarshal.GetReference(source);
+            ref ushort destinationBase = ref MemoryMarshal.GetReference(destination);
+            uint leftWeight = (uint)(32 - fraction);
+            uint rightWeight = (uint)fraction;
+            int i = 0;
+
+            // Adjacent source vectors overlap by one sample, aligning each left/right reference pair in the same lane.
+            // Widening keeps the largest 12-bit Q5 weighted sum below the UInt32 limit before narrowing to sample storage.
+            if (Vector512.IsHardwareAccelerated)
+            {
+                int oneVectorFromEnd = destination.Length - Vector512<ushort>.Count;
+                for (; i <= oneVectorFromEnd; i += Vector512<ushort>.Count)
+                {
+                    Vector512<ushort> left = Vector512.LoadUnsafe(ref sourceBase, (nuint)i);
+                    Vector512<ushort> right = Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + 1));
+                    (Vector512<uint> leftLow, Vector512<uint> leftHigh) = Vector512.Widen(left);
+                    (Vector512<uint> rightLow, Vector512<uint> rightHigh) = Vector512.Widen(right);
+                    Vector512<uint> low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector512.Create(16U)) >> 5;
+                    Vector512<uint> high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector512.Create(16U)) >> 5;
+                    Vector512.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i));
+                }
+            }
+
+            if (Vector256.IsHardwareAccelerated)
+            {
+                int oneVectorFromEnd = destination.Length - Vector256<ushort>.Count;
+                for (; i <= oneVectorFromEnd; i += Vector256<ushort>.Count)
+                {
+                    Vector256<ushort> left = Vector256.LoadUnsafe(ref sourceBase, (nuint)i);
+                    Vector256<ushort> right = Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + 1));
+                    (Vector256<uint> leftLow, Vector256<uint> leftHigh) = Vector256.Widen(left);
+                    (Vector256<uint> rightLow, Vector256<uint> rightHigh) = Vector256.Widen(right);
+                    Vector256<uint> low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector256.Create(16U)) >> 5;
+                    Vector256<uint> high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector256.Create(16U)) >> 5;
+                    Vector256.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i));
+                }
+            }
+
+            if (Vector128.IsHardwareAccelerated)
+            {
+                int oneVectorFromEnd = destination.Length - Vector128<ushort>.Count;
+                for (; i <= oneVectorFromEnd; i += Vector128<ushort>.Count)
+                {
+                    Vector128<ushort> left = Vector128.LoadUnsafe(ref sourceBase, (nuint)i);
+                    Vector128<ushort> right = Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + 1));
+                    (Vector128<uint> leftLow, Vector128<uint> leftHigh) = Vector128.Widen(left);
+                    (Vector128<uint> rightLow, Vector128<uint> rightHigh) = Vector128.Widen(right);
+                    Vector128<uint> low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector128.Create(16U)) >> 5;
+                    Vector128<uint> high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector128.Create(16U)) >> 5;
+                    Vector128.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i));
+                }
+            }
+
+            for (; i < destination.Length; i++)
+            {
+                Unsafe.Add(ref destinationBase, i) = (ushort)(((source[i] * leftWeight) + (source[i + 1] * rightWeight) + 16) >> 5);
+            }
+        }
+
+        /// <summary>
+        /// Transposes a square horizontal prediction block into the reconstructed destination.
+        /// </summary>
+        /// <param name="source">The contiguous transposed prediction block.</param>
+        /// <param name="destination">The destination block origin.</param>
+        /// <param name="destinationStride">The destination row stride.</param>
+        /// <param name="size">The square block side.</param>
+        private static void TransposeBlock(ReadOnlySpan<ushort> source, Span<ushort> destination, int destinationStride, int size)
+        {
+            if (Vector128.IsHardwareAccelerated && size >= Vector128<ushort>.Count)
+            {
+                for (int y = 0; y < size; y += Vector128<ushort>.Count)
+                {
+                    for (int x = 0; x < size; x += Vector128<ushort>.Count)
+                    {
+                        Transpose8x8(source, destination, destinationStride, size, x, y);
+                    }
+                }
+
+                return;
+            }
+
+            for (int y = 0; y < size; y++)
+            {
+                for (int x = 0; x < size; x++)
+                {
+                    destination[(x * destinationStride) + y] = source[(y * size) + x];
+                }
+            }
+        }
+
+        /// <summary>
+        /// Transposes one eight-by-eight tile of 16-bit prediction samples.
+        /// </summary>
+        /// <param name="source">The contiguous source block.</param>
+        /// <param name="destination">The destination block origin.</param>
+        /// <param name="destinationStride">The destination row stride.</param>
+        /// <param name="sourceStride">The contiguous source row stride.</param>
+        /// <param name="x">The tile X coordinate in the source block.</param>
+        /// <param name="y">The tile Y coordinate in the source block.</param>
+        private static void Transpose8x8(
+            ReadOnlySpan<ushort> source,
+            Span<ushort> destination,
+            int destinationStride,
+            int sourceStride,
+            int x,
+            int y)
+        {
+            ref ushort sourceBase = ref MemoryMarshal.GetReference(source);
+            ref ushort destinationBase = ref MemoryMarshal.GetReference(destination);
+            Vector128<short> row0 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 0) * sourceStride) + x)).AsInt16();
+            Vector128<short> row1 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 1) * sourceStride) + x)).AsInt16();
+            Vector128<short> row2 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 2) * sourceStride) + x)).AsInt16();
+            Vector128<short> row3 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 3) * sourceStride) + x)).AsInt16();
+            Vector128<short> row4 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 4) * sourceStride) + x)).AsInt16();
+            Vector128<short> row5 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 5) * sourceStride) + x)).AsInt16();
+            Vector128<short> row6 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 6) * sourceStride) + x)).AsInt16();
+            Vector128<short> row7 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 7) * sourceStride) + x)).AsInt16();
+
+            // Three zip stages exchange one, two, then four 16-bit coordinates. The resulting vectors are the eight
+            // source columns in row order, so each can be stored contiguously into one destination row.
+            Vector128<short> pair0 = Vector128_.UnpackLow(row0, row1);
+            Vector128<short> pair1 = Vector128_.UnpackHigh(row0, row1);
+            Vector128<short> pair2 = Vector128_.UnpackLow(row2, row3);
+            Vector128<short> pair3 = Vector128_.UnpackHigh(row2, row3);
+            Vector128<short> pair4 = Vector128_.UnpackLow(row4, row5);
+            Vector128<short> pair5 = Vector128_.UnpackHigh(row4, row5);
+            Vector128<short> pair6 = Vector128_.UnpackLow(row6, row7);
+            Vector128<short> pair7 = Vector128_.UnpackHigh(row6, row7);
+            Vector128<int> quad0 = Vector128_.UnpackLow(pair0.AsInt32(), pair2.AsInt32());
+            Vector128<int> quad1 = Vector128_.UnpackHigh(pair0.AsInt32(), pair2.AsInt32());
+            Vector128<int> quad2 = Vector128_.UnpackLow(pair1.AsInt32(), pair3.AsInt32());
+            Vector128<int> quad3 = Vector128_.UnpackHigh(pair1.AsInt32(), pair3.AsInt32());
+            Vector128<int> quad4 = Vector128_.UnpackLow(pair4.AsInt32(), pair6.AsInt32());
+            Vector128<int> quad5 = Vector128_.UnpackHigh(pair4.AsInt32(), pair6.AsInt32());
+            Vector128<int> quad6 = Vector128_.UnpackLow(pair5.AsInt32(), pair7.AsInt32());
+            Vector128<int> quad7 = Vector128_.UnpackHigh(pair5.AsInt32(), pair7.AsInt32());
+            Vector128<ushort> column0 = Vector128_.UnpackLow(quad0.AsInt64(), quad4.AsInt64()).AsUInt16();
+            Vector128<ushort> column1 = Vector128_.UnpackHigh(quad0.AsInt64(), quad4.AsInt64()).AsUInt16();
+            Vector128<ushort> column2 = Vector128_.UnpackLow(quad1.AsInt64(), quad5.AsInt64()).AsUInt16();
+            Vector128<ushort> column3 = Vector128_.UnpackHigh(quad1.AsInt64(), quad5.AsInt64()).AsUInt16();
+            Vector128<ushort> column4 = Vector128_.UnpackLow(quad2.AsInt64(), quad6.AsInt64()).AsUInt16();
+            Vector128<ushort> column5 = Vector128_.UnpackHigh(quad2.AsInt64(), quad6.AsInt64()).AsUInt16();
+            Vector128<ushort> column6 = Vector128_.UnpackLow(quad3.AsInt64(), quad7.AsInt64()).AsUInt16();
+            Vector128<ushort> column7 = Vector128_.UnpackHigh(quad3.AsInt64(), quad7.AsInt64()).AsUInt16();
+            column0.StoreUnsafe(ref destinationBase, (nuint)(((x + 0) * destinationStride) + y));
+            column1.StoreUnsafe(ref destinationBase, (nuint)(((x + 1) * destinationStride) + y));
+            column2.StoreUnsafe(ref destinationBase, (nuint)(((x + 2) * destinationStride) + y));
+            column3.StoreUnsafe(ref destinationBase, (nuint)(((x + 3) * destinationStride) + y));
+            column4.StoreUnsafe(ref destinationBase, (nuint)(((x + 4) * destinationStride) + y));
+            column5.StoreUnsafe(ref destinationBase, (nuint)(((x + 5) * destinationStride) + y));
+            column6.StoreUnsafe(ref destinationBase, (nuint)(((x + 6) * destinationStride) + y));
+            column7.StoreUnsafe(ref destinationBase, (nuint)(((x + 7) * destinationStride) + y));
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.DcOperator.cs

@@ -0,0 +1,108 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines DC intra-prediction arithmetic.
+/// </content>
+internal static partial class HevcIntraPredictor
+{
+    /// <summary>
+    /// Implements DC prediction and its optional luma boundary filter.
+    /// </summary>
+    private readonly struct DcOperator : IHevcIntraPredictionOperator
+    {
+        /// <inheritdoc/>
+        public static void Predict(
+            ReadOnlySpan<ushort> top,
+            ReadOnlySpan<ushort> left,
+            Span<ushort> destination,
+            int destinationStride,
+            int size,
+            int mode,
+            int bitDepth,
+            bool filterPredictionEdges,
+            Span<ushort> scratch)
+        {
+            uint sum = SumSamples(top.Slice(1, size)) + SumSamples(left.Slice(1, size));
+            ushort dc = (ushort)((sum + (uint)size) >> (BitOperations.Log2((uint)size) + 1));
+            for (int y = 0; y < size; y++)
+            {
+                destination.Slice(y * destinationStride, size).Fill(dc);
+            }
+
+            if (!filterPredictionEdges)
+            {
+                return;
+            }
+
+            destination[0] = (ushort)((top[1] + left[1] + (2 * dc) + 2) >> 2);
+            for (int x = 1; x < size; x++)
+            {
+                destination[x] = (ushort)((top[x + 1] + (3 * dc) + 2) >> 2);
+            }
+
+            for (int y = 1; y < size; y++)
+            {
+                destination[y * destinationStride] = (ushort)((left[y + 1] + (3 * dc) + 2) >> 2);
+            }
+        }
+
+        /// <summary>
+        /// Sums reconstructed reference samples without overflowing their 16-bit storage.
+        /// </summary>
+        /// <param name="samples">The samples to sum.</param>
+        /// <returns>The exact unsigned sum.</returns>
+        private static uint SumSamples(ReadOnlySpan<ushort> samples)
+        {
+            ref ushort samplesBase = ref MemoryMarshal.GetReference(samples);
+            uint sum = 0;
+            int i = 0;
+
+            // Widen before reduction because a complete 64-sample, 12-bit reference edge exceeds UInt16. The shared index
+            // lets narrower vectors consume only the remainder from the widest available path.
+            if (Vector512.IsHardwareAccelerated)
+            {
+                int oneVectorFromEnd = samples.Length - Vector512<ushort>.Count;
+                for (; i <= oneVectorFromEnd; i += Vector512<ushort>.Count)
+                {
+                    (Vector512<uint> low, Vector512<uint> high) = Vector512.Widen(Vector512.LoadUnsafe(ref samplesBase, (nuint)i));
+                    sum += Vector512.Sum(low) + Vector512.Sum(high);
+                }
+            }
+
+            if (Vector256.IsHardwareAccelerated)
+            {
+                int oneVectorFromEnd = samples.Length - Vector256<ushort>.Count;
+                for (; i <= oneVectorFromEnd; i += Vector256<ushort>.Count)
+                {
+                    (Vector256<uint> low, Vector256<uint> high) = Vector256.Widen(Vector256.LoadUnsafe(ref samplesBase, (nuint)i));
+                    sum += Vector256.Sum(low) + Vector256.Sum(high);
+                }
+            }
+
+            if (Vector128.IsHardwareAccelerated)
+            {
+                int oneVectorFromEnd = samples.Length - Vector128<ushort>.Count;
+                for (; i <= oneVectorFromEnd; i += Vector128<ushort>.Count)
+                {
+                    (Vector128<uint> low, Vector128<uint> high) = Vector128.Widen(Vector128.LoadUnsafe(ref samplesBase, (nuint)i));
+                    sum += Vector128.Sum(low) + Vector128.Sum(high);
+                }
+            }
+
+            for (; i < samples.Length; i++)
+            {
+                sum += Unsafe.Add(ref samplesBase, i);
+            }
+
+            return sum;
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operations.cs

@@ -1,431 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Numerics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Runtime.Intrinsics;
-using SixLabors.ImageSharp.Common.Helpers;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
-
-/// <content>
-/// Provides shared SIMD operations used by the closed prediction operators. Each lane represents one output column;
-/// planar and angular interpolation widen 16-bit references before their Q5 weighted sums, then narrow only after the
-/// normative rounding shift. Horizontal prediction reuses the vertical row kernel through a caller-owned contiguous
-/// block and an eight-by-eight transpose, keeping the arithmetic identical without gathering strided destination rows.
-/// </content>
-internal static partial class HevcIntraPredictor
-{
-    /// <summary>
-    /// Calculates one 512-bit half of a planar prediction row.
-    /// </summary>
-    /// <param name="top">The top reference samples.</param>
-    /// <param name="indices">The zero-based X coordinates.</param>
-    /// <param name="left">The left reference sample for the row.</param>
-    /// <param name="topRight">The top-right reference sample.</param>
-    /// <param name="bottomLeft">The bottom-left reference sample.</param>
-    /// <param name="topWeight">The top-reference weight.</param>
-    /// <param name="bottomWeight">The bottom-left-reference weight.</param>
-    /// <param name="size">The square block side.</param>
-    /// <param name="rounding">The division rounding constant.</param>
-    /// <param name="shift">The division shift.</param>
-    /// <returns>The predicted samples as widened lanes.</returns>
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static Vector512<uint> CalculatePlanarVector(
-        Vector512<uint> top,
-        Vector512<uint> indices,
-        uint left,
-        uint topRight,
-        uint bottomLeft,
-        uint topWeight,
-        uint bottomWeight,
-        uint size,
-        uint rounding,
-        int shift)
-    {
-        Vector512<uint> horizontal = ((Vector512.Create(size - 1) - indices) * left) + ((indices + Vector512<uint>.One) * topRight);
-        Vector512<uint> vertical = (top * topWeight) + Vector512.Create(bottomLeft * bottomWeight);
-        return (horizontal + vertical + Vector512.Create(rounding)) >> shift;
-    }
-
-    /// <summary>
-    /// Calculates one 256-bit half of a planar prediction row.
-    /// </summary>
-    /// <param name="top">The top reference samples.</param>
-    /// <param name="indices">The zero-based X coordinates.</param>
-    /// <param name="left">The left reference sample for the row.</param>
-    /// <param name="topRight">The top-right reference sample.</param>
-    /// <param name="bottomLeft">The bottom-left reference sample.</param>
-    /// <param name="topWeight">The top-reference weight.</param>
-    /// <param name="bottomWeight">The bottom-left-reference weight.</param>
-    /// <param name="size">The square block side.</param>
-    /// <param name="rounding">The division rounding constant.</param>
-    /// <param name="shift">The division shift.</param>
-    /// <returns>The predicted samples as widened lanes.</returns>
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static Vector256<uint> CalculatePlanarVector(
-        Vector256<uint> top,
-        Vector256<uint> indices,
-        uint left,
-        uint topRight,
-        uint bottomLeft,
-        uint topWeight,
-        uint bottomWeight,
-        uint size,
-        uint rounding,
-        int shift)
-    {
-        Vector256<uint> horizontal = ((Vector256.Create(size - 1) - indices) * left) + ((indices + Vector256<uint>.One) * topRight);
-        Vector256<uint> vertical = (top * topWeight) + Vector256.Create(bottomLeft * bottomWeight);
-        return (horizontal + vertical + Vector256.Create(rounding)) >> shift;
-    }
-
-    /// <summary>
-    /// Calculates one 128-bit half of a planar prediction row.
-    /// </summary>
-    /// <param name="top">The top reference samples.</param>
-    /// <param name="indices">The zero-based X coordinates.</param>
-    /// <param name="left">The left reference sample for the row.</param>
-    /// <param name="topRight">The top-right reference sample.</param>
-    /// <param name="bottomLeft">The bottom-left reference sample.</param>
-    /// <param name="topWeight">The top-reference weight.</param>
-    /// <param name="bottomWeight">The bottom-left-reference weight.</param>
-    /// <param name="size">The square block side.</param>
-    /// <param name="rounding">The division rounding constant.</param>
-    /// <param name="shift">The division shift.</param>
-    /// <returns>The predicted samples as widened lanes.</returns>
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static Vector128<uint> CalculatePlanarVector(
-        Vector128<uint> top,
-        Vector128<uint> indices,
-        uint left,
-        uint topRight,
-        uint bottomLeft,
-        uint topWeight,
-        uint bottomWeight,
-        uint size,
-        uint rounding,
-        int shift)
-    {
-        Vector128<uint> horizontal = ((Vector128.Create(size - 1) - indices) * left) + ((indices + Vector128<uint>.One) * topRight);
-        Vector128<uint> vertical = (top * topWeight) + Vector128.Create(bottomLeft * bottomWeight);
-        return (horizontal + vertical + Vector128.Create(rounding)) >> shift;
-    }
-
-    /// <summary>
-    /// Sums reconstructed reference samples without overflowing their 16-bit storage.
-    /// </summary>
-    /// <param name="samples">The samples to sum.</param>
-    /// <returns>The exact unsigned sum.</returns>
-    private static uint SumSamples(ReadOnlySpan<ushort> samples)
-    {
-        ref ushort samplesBase = ref MemoryMarshal.GetReference(samples);
-        uint sum = 0;
-        int i = 0;
-
-        // Widen before reduction because a complete 64-sample, 12-bit reference edge exceeds UInt16. The shared index
-        // lets narrower vectors consume only the remainder from the widest available path.
-        if (Vector512.IsHardwareAccelerated)
-        {
-            int oneVectorFromEnd = samples.Length - Vector512<ushort>.Count;
-            for (; i <= oneVectorFromEnd; i += Vector512<ushort>.Count)
-            {
-                (Vector512<uint> low, Vector512<uint> high) = Vector512.Widen(Vector512.LoadUnsafe(ref samplesBase, (nuint)i));
-                sum += Vector512.Sum(low) + Vector512.Sum(high);
-            }
-        }
-
-        if (Vector256.IsHardwareAccelerated)
-        {
-            int oneVectorFromEnd = samples.Length - Vector256<ushort>.Count;
-            for (; i <= oneVectorFromEnd; i += Vector256<ushort>.Count)
-            {
-                (Vector256<uint> low, Vector256<uint> high) = Vector256.Widen(Vector256.LoadUnsafe(ref samplesBase, (nuint)i));
-                sum += Vector256.Sum(low) + Vector256.Sum(high);
-            }
-        }
-
-        if (Vector128.IsHardwareAccelerated)
-        {
-            int oneVectorFromEnd = samples.Length - Vector128<ushort>.Count;
-            for (; i <= oneVectorFromEnd; i += Vector128<ushort>.Count)
-            {
-                (Vector128<uint> low, Vector128<uint> high) = Vector128.Widen(Vector128.LoadUnsafe(ref samplesBase, (nuint)i));
-                sum += Vector128.Sum(low) + Vector128.Sum(high);
-            }
-        }
-
-        for (; i < samples.Length; i++)
-        {
-            sum += Unsafe.Add(ref samplesBase, i);
-        }
-
-        return sum;
-    }
-
-    /// <summary>
-    /// Copies the top reference into every row and optionally filters the first column.
-    /// </summary>
-    /// <param name="top">The top reference samples.</param>
-    /// <param name="left">The left reference samples.</param>
-    /// <param name="destination">The destination block origin.</param>
-    /// <param name="destinationStride">The destination row stride.</param>
-    /// <param name="size">The square block side.</param>
-    /// <param name="bitDepth">The reconstructed component precision.</param>
-    /// <param name="filterPredictionEdges">Whether the vertical luma edge filter applies.</param>
-    private static void PredictVertical(
-        ReadOnlySpan<ushort> top,
-        ReadOnlySpan<ushort> left,
-        Span<ushort> destination,
-        int destinationStride,
-        int size,
-        int bitDepth,
-        bool filterPredictionEdges)
-    {
-        ReadOnlySpan<ushort> row = top.Slice(1, size);
-        int maximum = (1 << bitDepth) - 1;
-        for (int y = 0; y < size; y++)
-        {
-            row.CopyTo(destination.Slice(y * destinationStride, size));
-            if (filterPredictionEdges)
-            {
-                int sample = destination[y * destinationStride] + ((left[y + 1] - left[0]) >> 1);
-                destination[y * destinationStride] = (ushort)Math.Clamp(sample, 0, maximum);
-            }
-        }
-    }
-
-    /// <summary>
-    /// Fills each row from its left reference and optionally filters the first row.
-    /// </summary>
-    /// <param name="top">The top reference samples.</param>
-    /// <param name="left">The left reference samples.</param>
-    /// <param name="destination">The destination block origin.</param>
-    /// <param name="destinationStride">The destination row stride.</param>
-    /// <param name="size">The square block side.</param>
-    /// <param name="bitDepth">The reconstructed component precision.</param>
-    /// <param name="filterPredictionEdges">Whether the horizontal luma edge filter applies.</param>
-    private static void PredictHorizontal(
-        ReadOnlySpan<ushort> top,
-        ReadOnlySpan<ushort> left,
-        Span<ushort> destination,
-        int destinationStride,
-        int size,
-        int bitDepth,
-        bool filterPredictionEdges)
-    {
-        for (int y = 0; y < size; y++)
-        {
-            destination.Slice(y * destinationStride, size).Fill(left[y + 1]);
-        }
-
-        if (!filterPredictionEdges)
-        {
-            return;
-        }
-
-        int maximum = (1 << bitDepth) - 1;
-        for (int x = 0; x < size; x++)
-        {
-            int sample = destination[x] + ((top[x + 1] - top[0]) >> 1);
-            destination[x] = (ushort)Math.Clamp(sample, 0, maximum);
-        }
-    }
-
-    /// <summary>
-    /// Generates a vertical-oriented angular block using contiguous SIMD interpolation within each row.
-    /// </summary>
-    /// <param name="main">The main reference beginning at logical index zero.</param>
-    /// <param name="mainOrigin">The span index corresponding to logical reference index zero.</param>
-    /// <param name="destination">The contiguous destination or transposition scratch block.</param>
-    /// <param name="destinationStride">The destination row stride.</param>
-    /// <param name="size">The square block side.</param>
-    /// <param name="angle">The signed prediction displacement in thirty-second-sample units.</param>
-    private static void PredictAngularRows(
-        ReadOnlySpan<ushort> main,
-        int mainOrigin,
-        Span<ushort> destination,
-        int destinationStride,
-        int size,
-        int angle)
-    {
-        for (int y = 0, deltaPosition = angle; y < size; y++, deltaPosition += angle)
-        {
-            int deltaInteger = deltaPosition >> 5;
-            int deltaFraction = deltaPosition & 31;
-            int sourceOffset = mainOrigin + deltaInteger + 1;
-            Span<ushort> row = destination.Slice(y * destinationStride, size);
-            if (deltaFraction == 0)
-            {
-                main.Slice(sourceOffset, size).CopyTo(row);
-            }
-            else
-            {
-                InterpolateAngularRow(main[sourceOffset..], row, deltaFraction);
-            }
-        }
-    }
-
-    /// <summary>
-    /// Interpolates one angular prediction row between consecutive main-reference samples.
-    /// </summary>
-    /// <param name="source">The first main-reference sample for the row.</param>
-    /// <param name="destination">The destination prediction row.</param>
-    /// <param name="fraction">The right-hand weight with a denominator of thirty-two.</param>
-    private static void InterpolateAngularRow(ReadOnlySpan<ushort> source, Span<ushort> destination, int fraction)
-    {
-        ref ushort sourceBase = ref MemoryMarshal.GetReference(source);
-        ref ushort destinationBase = ref MemoryMarshal.GetReference(destination);
-        uint leftWeight = (uint)(32 - fraction);
-        uint rightWeight = (uint)fraction;
-        int i = 0;
-
-        // Adjacent source vectors overlap by one sample, aligning each left/right reference pair in the same lane.
-        // Widening keeps the largest 12-bit Q5 weighted sum below the UInt32 limit before narrowing to sample storage.
-        if (Vector512.IsHardwareAccelerated)
-        {
-            int oneVectorFromEnd = destination.Length - Vector512<ushort>.Count;
-            for (; i <= oneVectorFromEnd; i += Vector512<ushort>.Count)
-            {
-                Vector512<ushort> left = Vector512.LoadUnsafe(ref sourceBase, (nuint)i);
-                Vector512<ushort> right = Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + 1));
-                (Vector512<uint> leftLow, Vector512<uint> leftHigh) = Vector512.Widen(left);
-                (Vector512<uint> rightLow, Vector512<uint> rightHigh) = Vector512.Widen(right);
-                Vector512<uint> low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector512.Create(16U)) >> 5;
-                Vector512<uint> high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector512.Create(16U)) >> 5;
-                Vector512.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i));
-            }
-        }
-
-        if (Vector256.IsHardwareAccelerated)
-        {
-            int oneVectorFromEnd = destination.Length - Vector256<ushort>.Count;
-            for (; i <= oneVectorFromEnd; i += Vector256<ushort>.Count)
-            {
-                Vector256<ushort> left = Vector256.LoadUnsafe(ref sourceBase, (nuint)i);
-                Vector256<ushort> right = Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + 1));
-                (Vector256<uint> leftLow, Vector256<uint> leftHigh) = Vector256.Widen(left);
-                (Vector256<uint> rightLow, Vector256<uint> rightHigh) = Vector256.Widen(right);
-                Vector256<uint> low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector256.Create(16U)) >> 5;
-                Vector256<uint> high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector256.Create(16U)) >> 5;
-                Vector256.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i));
-            }
-        }
-
-        if (Vector128.IsHardwareAccelerated)
-        {
-            int oneVectorFromEnd = destination.Length - Vector128<ushort>.Count;
-            for (; i <= oneVectorFromEnd; i += Vector128<ushort>.Count)
-            {
-                Vector128<ushort> left = Vector128.LoadUnsafe(ref sourceBase, (nuint)i);
-                Vector128<ushort> right = Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + 1));
-                (Vector128<uint> leftLow, Vector128<uint> leftHigh) = Vector128.Widen(left);
-                (Vector128<uint> rightLow, Vector128<uint> rightHigh) = Vector128.Widen(right);
-                Vector128<uint> low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector128.Create(16U)) >> 5;
-                Vector128<uint> high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector128.Create(16U)) >> 5;
-                Vector128.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i));
-            }
-        }
-
-        for (; i < destination.Length; i++)
-        {
-            Unsafe.Add(ref destinationBase, i) = (ushort)(((source[i] * leftWeight) + (source[i + 1] * rightWeight) + 16) >> 5);
-        }
-    }
-
-    /// <summary>
-    /// Transposes a square horizontal prediction block into the reconstructed destination.
-    /// </summary>
-    /// <param name="source">The contiguous transposed prediction block.</param>
-    /// <param name="destination">The destination block origin.</param>
-    /// <param name="destinationStride">The destination row stride.</param>
-    /// <param name="size">The square block side.</param>
-    private static void TransposeBlock(ReadOnlySpan<ushort> source, Span<ushort> destination, int destinationStride, int size)
-    {
-        if (Vector128.IsHardwareAccelerated && size >= Vector128<ushort>.Count)
-        {
-            for (int y = 0; y < size; y += Vector128<ushort>.Count)
-            {
-                for (int x = 0; x < size; x += Vector128<ushort>.Count)
-                {
-                    Transpose8x8(source, destination, destinationStride, size, x, y);
-                }
-            }
-
-            return;
-        }
-
-        for (int y = 0; y < size; y++)
-        {
-            for (int x = 0; x < size; x++)
-            {
-                destination[(x * destinationStride) + y] = source[(y * size) + x];
-            }
-        }
-    }
-
-    /// <summary>
-    /// Transposes one eight-by-eight tile of 16-bit prediction samples.
-    /// </summary>
-    /// <param name="source">The contiguous source block.</param>
-    /// <param name="destination">The destination block origin.</param>
-    /// <param name="destinationStride">The destination row stride.</param>
-    /// <param name="sourceStride">The contiguous source row stride.</param>
-    /// <param name="x">The tile X coordinate in the source block.</param>
-    /// <param name="y">The tile Y coordinate in the source block.</param>
-    private static void Transpose8x8(
-        ReadOnlySpan<ushort> source,
-        Span<ushort> destination,
-        int destinationStride,
-        int sourceStride,
-        int x,
-        int y)
-    {
-        ref ushort sourceBase = ref MemoryMarshal.GetReference(source);
-        ref ushort destinationBase = ref MemoryMarshal.GetReference(destination);
-        Vector128<short> row0 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 0) * sourceStride) + x)).AsInt16();
-        Vector128<short> row1 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 1) * sourceStride) + x)).AsInt16();
-        Vector128<short> row2 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 2) * sourceStride) + x)).AsInt16();
-        Vector128<short> row3 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 3) * sourceStride) + x)).AsInt16();
-        Vector128<short> row4 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 4) * sourceStride) + x)).AsInt16();
-        Vector128<short> row5 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 5) * sourceStride) + x)).AsInt16();
-        Vector128<short> row6 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 6) * sourceStride) + x)).AsInt16();
-        Vector128<short> row7 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 7) * sourceStride) + x)).AsInt16();
-
-        // Three zip stages exchange one, two, then four 16-bit coordinates. The resulting vectors are the eight
-        // source columns in row order, so each can be stored contiguously into one destination row.
-        Vector128<short> pair0 = Vector128_.UnpackLow(row0, row1);
-        Vector128<short> pair1 = Vector128_.UnpackHigh(row0, row1);
-        Vector128<short> pair2 = Vector128_.UnpackLow(row2, row3);
-        Vector128<short> pair3 = Vector128_.UnpackHigh(row2, row3);
-        Vector128<short> pair4 = Vector128_.UnpackLow(row4, row5);
-        Vector128<short> pair5 = Vector128_.UnpackHigh(row4, row5);
-        Vector128<short> pair6 = Vector128_.UnpackLow(row6, row7);
-        Vector128<short> pair7 = Vector128_.UnpackHigh(row6, row7);
-        Vector128<int> quad0 = Vector128_.UnpackLow(pair0.AsInt32(), pair2.AsInt32());
-        Vector128<int> quad1 = Vector128_.UnpackHigh(pair0.AsInt32(), pair2.AsInt32());
-        Vector128<int> quad2 = Vector128_.UnpackLow(pair1.AsInt32(), pair3.AsInt32());
-        Vector128<int> quad3 = Vector128_.UnpackHigh(pair1.AsInt32(), pair3.AsInt32());
-        Vector128<int> quad4 = Vector128_.UnpackLow(pair4.AsInt32(), pair6.AsInt32());
-        Vector128<int> quad5 = Vector128_.UnpackHigh(pair4.AsInt32(), pair6.AsInt32());
-        Vector128<int> quad6 = Vector128_.UnpackLow(pair5.AsInt32(), pair7.AsInt32());
-        Vector128<int> quad7 = Vector128_.UnpackHigh(pair5.AsInt32(), pair7.AsInt32());
-        Vector128<ushort> column0 = Vector128_.UnpackLow(quad0.AsInt64(), quad4.AsInt64()).AsUInt16();
-        Vector128<ushort> column1 = Vector128_.UnpackHigh(quad0.AsInt64(), quad4.AsInt64()).AsUInt16();
-        Vector128<ushort> column2 = Vector128_.UnpackLow(quad1.AsInt64(), quad5.AsInt64()).AsUInt16();
-        Vector128<ushort> column3 = Vector128_.UnpackHigh(quad1.AsInt64(), quad5.AsInt64()).AsUInt16();
-        Vector128<ushort> column4 = Vector128_.UnpackLow(quad2.AsInt64(), quad6.AsInt64()).AsUInt16();
-        Vector128<ushort> column5 = Vector128_.UnpackHigh(quad2.AsInt64(), quad6.AsInt64()).AsUInt16();
-        Vector128<ushort> column6 = Vector128_.UnpackLow(quad3.AsInt64(), quad7.AsInt64()).AsUInt16();
-        Vector128<ushort> column7 = Vector128_.UnpackHigh(quad3.AsInt64(), quad7.AsInt64()).AsUInt16();
-        column0.StoreUnsafe(ref destinationBase, (nuint)(((x + 0) * destinationStride) + y));
-        column1.StoreUnsafe(ref destinationBase, (nuint)(((x + 1) * destinationStride) + y));
-        column2.StoreUnsafe(ref destinationBase, (nuint)(((x + 2) * destinationStride) + y));
-        column3.StoreUnsafe(ref destinationBase, (nuint)(((x + 3) * destinationStride) + y));
-        column4.StoreUnsafe(ref destinationBase, (nuint)(((x + 4) * destinationStride) + y));
-        column5.StoreUnsafe(ref destinationBase, (nuint)(((x + 5) * destinationStride) + y));
-        column6.StoreUnsafe(ref destinationBase, (nuint)(((x + 6) * destinationStride) + y));
-        column7.StoreUnsafe(ref destinationBase, (nuint)(((x + 7) * destinationStride) + y));
-    }
-}

src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operator.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines the HEVC intra-prediction operator contract.
+/// </content>
+internal static partial class HevcIntraPredictor
+{
+    /// <summary>
+    /// Defines one closed intra-prediction operation selected by the decoded mode.
+    /// </summary>
+    private interface IHevcIntraPredictionOperator
+    {
+        /// <summary>
+        /// Reconstructs one square prediction block.
+        /// </summary>
+        /// <param name="top">The top-left, top, and top-right reference samples.</param>
+        /// <param name="left">The top-left, left, and below-left reference samples.</param>
+        /// <param name="destination">The destination buffer beginning at the block origin.</param>
+        /// <param name="destinationStride">The destination row stride in samples.</param>
+        /// <param name="size">The square block side in samples.</param>
+        /// <param name="mode">The decoded prediction mode.</param>
+        /// <param name="bitDepth">The reconstructed component precision.</param>
+        /// <param name="filterPredictionEdges">Whether the luma edge filter applies to the selected block.</param>
+        /// <param name="scratch">The caller-owned block and extended-reference scratch space.</param>
+        public static abstract void Predict(
+            ReadOnlySpan<ushort> top,
+            ReadOnlySpan<ushort> left,
+            Span<ushort> destination,
+            int destinationStride,
+            int size,
+            int mode,
+            int bitDepth,
+            bool filterPredictionEdges,
+            Span<ushort> scratch);
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs → src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.PlanarOperator.cs

@@ -9,14 +9,14 @@ using System.Runtime.Intrinsics;
 namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
 
 /// <content>
-/// Provides the closed planar, DC, and angular prediction operators.
+/// Defines planar intra-prediction arithmetic.
 /// </content>
 internal static partial class HevcIntraPredictor
 {
     /// <summary>
     /// Implements planar interpolation between the top, left, bottom-left, and top-right references.
     /// </summary>
-    private readonly struct PlanarPredictionOperator : IHevcIntraPredictionOperator<PlanarPredictionOperator>
+    private readonly struct PlanarOperator : IHevcIntraPredictionOperator
     {
         /// <inheritdoc/>
         public static void Predict(
@@ -171,112 +171,101 @@ internal static partial class HevcIntraPredictor
                 }
             }
         }
-    }
 
-    /// <summary>
-    /// Implements DC prediction and its optional luma boundary filter.
-    /// </summary>
-    private readonly struct DcPredictionOperator : IHevcIntraPredictionOperator<DcPredictionOperator>
-    {
-        /// <inheritdoc/>
-        public static void Predict(
-            ReadOnlySpan<ushort> top,
-            ReadOnlySpan<ushort> left,
-            Span<ushort> destination,
-            int destinationStride,
-            int size,
-            int mode,
-            int bitDepth,
-            bool filterPredictionEdges,
-            Span<ushort> scratch)
+        /// <summary>
+        /// Calculates one 512-bit half of a planar prediction row.
+        /// </summary>
+        /// <param name="top">The top reference samples.</param>
+        /// <param name="indices">The zero-based X coordinates.</param>
+        /// <param name="left">The left reference sample for the row.</param>
+        /// <param name="topRight">The top-right reference sample.</param>
+        /// <param name="bottomLeft">The bottom-left reference sample.</param>
+        /// <param name="topWeight">The top-reference weight.</param>
+        /// <param name="bottomWeight">The bottom-left-reference weight.</param>
+        /// <param name="size">The square block side.</param>
+        /// <param name="rounding">The division rounding constant.</param>
+        /// <param name="shift">The division shift.</param>
+        /// <returns>The predicted samples as widened lanes.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector512<uint> CalculatePlanarVector(
+            Vector512<uint> top,
+            Vector512<uint> indices,
+            uint left,
+            uint topRight,
+            uint bottomLeft,
+            uint topWeight,
+            uint bottomWeight,
+            uint size,
+            uint rounding,
+            int shift)
         {
-            uint sum = SumSamples(top.Slice(1, size)) + SumSamples(left.Slice(1, size));
-            ushort dc = (ushort)((sum + (uint)size) >> (BitOperations.Log2((uint)size) + 1));
-            for (int y = 0; y < size; y++)
-            {
-                destination.Slice(y * destinationStride, size).Fill(dc);
-            }
-
-            if (!filterPredictionEdges)
-            {
-                return;
-            }
-
-            destination[0] = (ushort)((top[1] + left[1] + (2 * dc) + 2) >> 2);
-            for (int x = 1; x < size; x++)
-            {
-                destination[x] = (ushort)((top[x + 1] + (3 * dc) + 2) >> 2);
-            }
-
-            for (int y = 1; y < size; y++)
-            {
-                destination[y * destinationStride] = (ushort)((left[y + 1] + (3 * dc) + 2) >> 2);
-            }
+            Vector512<uint> horizontal = ((Vector512.Create(size - 1) - indices) * left) + ((indices + Vector512<uint>.One) * topRight);
+            Vector512<uint> vertical = (top * topWeight) + Vector512.Create(bottomLeft * bottomWeight);
+            return (horizontal + vertical + Vector512.Create(rounding)) >> shift;
         }
-    }
 
-    /// <summary>
-    /// Implements the thirty-three directional intra-prediction modes.
-    /// </summary>
-    private readonly struct AngularPredictionOperator : IHevcIntraPredictionOperator<AngularPredictionOperator>
-    {
-        /// <inheritdoc/>
-        public static void Predict(
-            ReadOnlySpan<ushort> top,
-            ReadOnlySpan<ushort> left,
-            Span<ushort> destination,
-            int destinationStride,
-            int size,
-            int mode,
-            int bitDepth,
-            bool filterPredictionEdges,
-            Span<ushort> scratch)
+        /// <summary>
+        /// Calculates one 256-bit half of a planar prediction row.
+        /// </summary>
+        /// <param name="top">The top reference samples.</param>
+        /// <param name="indices">The zero-based X coordinates.</param>
+        /// <param name="left">The left reference sample for the row.</param>
+        /// <param name="topRight">The top-right reference sample.</param>
+        /// <param name="bottomLeft">The bottom-left reference sample.</param>
+        /// <param name="topWeight">The top-reference weight.</param>
+        /// <param name="bottomWeight">The bottom-left-reference weight.</param>
+        /// <param name="size">The square block side.</param>
+        /// <param name="rounding">The division rounding constant.</param>
+        /// <param name="shift">The division shift.</param>
+        /// <returns>The predicted samples as widened lanes.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector256<uint> CalculatePlanarVector(
+            Vector256<uint> top,
+            Vector256<uint> indices,
+            uint left,
+            uint topRight,
+            uint bottomLeft,
+            uint topWeight,
+            uint bottomWeight,
+            uint size,
+            uint rounding,
+            int shift)
         {
-            if (mode == VerticalMode)
-            {
-                PredictVertical(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges);
-                return;
-            }
-
-            if (mode == HorizontalMode)
-            {
-                PredictHorizontal(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges);
-                return;
-            }
-
-            bool vertical = mode >= FirstVerticalMode;
-            int angleMode = vertical ? mode - VerticalMode : HorizontalMode - mode;
-            int absoluteAngleMode = Math.Abs(angleMode);
-            int angle = PredictionAngles[absoluteAngleMode] * Math.Sign(angleMode);
-            ReadOnlySpan<ushort> main = vertical ? top : left;
-            ReadOnlySpan<ushort> side = vertical ? left : top;
-            Span<ushort> temporaryBlock = scratch[..(size * size)];
-            Span<ushort> extendedReference = scratch.Slice(size * size, (4 * size) + 1);
-            int mainOrigin = 0;
-
-            if (angle < 0)
-            {
-                mainOrigin = size * 2;
-                main[..(size + 1)].CopyTo(extendedReference[mainOrigin..]);
-                int inverseAngle = InversePredictionAngles[absoluteAngleMode];
-                int inverseAngleSum = 128;
-                int minimumIndex = (size * angle) >> 5;
-                for (int index = -1; index > minimumIndex; index--)
-                {
-                    inverseAngleSum += inverseAngle;
-                    extendedReference[mainOrigin + index] = side[inverseAngleSum >> 8];
-                }
-
-                main = extendedReference;
-            }
+            Vector256<uint> horizontal = ((Vector256.Create(size - 1) - indices) * left) + ((indices + Vector256<uint>.One) * topRight);
+            Vector256<uint> vertical = (top * topWeight) + Vector256.Create(bottomLeft * bottomWeight);
+            return (horizontal + vertical + Vector256.Create(rounding)) >> shift;
+        }
 
-            Span<ushort> prediction = vertical ? destination : temporaryBlock;
-            int predictionStride = vertical ? destinationStride : size;
-            PredictAngularRows(main, mainOrigin, prediction, predictionStride, size, angle);
-            if (!vertical)
-            {
-                TransposeBlock(temporaryBlock, destination, destinationStride, size);
-            }
+        /// <summary>
+        /// Calculates one 128-bit half of a planar prediction row.
+        /// </summary>
+        /// <param name="top">The top reference samples.</param>
+        /// <param name="indices">The zero-based X coordinates.</param>
+        /// <param name="left">The left reference sample for the row.</param>
+        /// <param name="topRight">The top-right reference sample.</param>
+        /// <param name="bottomLeft">The bottom-left reference sample.</param>
+        /// <param name="topWeight">The top-reference weight.</param>
+        /// <param name="bottomWeight">The bottom-left-reference weight.</param>
+        /// <param name="size">The square block side.</param>
+        /// <param name="rounding">The division rounding constant.</param>
+        /// <param name="shift">The division shift.</param>
+        /// <returns>The predicted samples as widened lanes.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector128<uint> CalculatePlanarVector(
+            Vector128<uint> top,
+            Vector128<uint> indices,
+            uint left,
+            uint topRight,
+            uint bottomLeft,
+            uint topWeight,
+            uint bottomWeight,
+            uint size,
+            uint rounding,
+            int shift)
+        {
+            Vector128<uint> horizontal = ((Vector128.Create(size - 1) - indices) * left) + ((indices + Vector128<uint>.One) * topRight);
+            Vector128<uint> vertical = (top * topWeight) + Vector128.Create(bottomLeft * bottomWeight);
+            return (horizontal + vertical + Vector128.Create(rounding)) >> shift;
         }
     }
 }

src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs

@@ -50,37 +50,6 @@ internal static partial class HevcIntraPredictor
     /// </summary>
     private const int MaximumBlockSize = 32;
 
-    /// <summary>
-    /// Defines one closed intra-prediction operation selected by the decoded mode.
-    /// </summary>
-    /// <typeparam name="TOperator">The implementing operator type.</typeparam>
-    private interface IHevcIntraPredictionOperator<TOperator>
-        where TOperator : struct, IHevcIntraPredictionOperator<TOperator>
-    {
-        /// <summary>
-        /// Reconstructs one square prediction block.
-        /// </summary>
-        /// <param name="top">The top-left, top, and top-right reference samples.</param>
-        /// <param name="left">The top-left, left, and below-left reference samples.</param>
-        /// <param name="destination">The destination buffer beginning at the block origin.</param>
-        /// <param name="destinationStride">The destination row stride in samples.</param>
-        /// <param name="size">The square block side in samples.</param>
-        /// <param name="mode">The decoded prediction mode.</param>
-        /// <param name="bitDepth">The reconstructed component precision.</param>
-        /// <param name="filterPredictionEdges">Whether the luma edge filter applies to the selected block.</param>
-        /// <param name="scratch">The caller-owned block and extended-reference scratch space.</param>
-        public static abstract void Predict(
-            ReadOnlySpan<ushort> top,
-            ReadOnlySpan<ushort> left,
-            Span<ushort> destination,
-            int destinationStride,
-            int size,
-            int mode,
-            int bitDepth,
-            bool filterPredictionEdges,
-            Span<ushort> scratch);
-    }
-
     /// <summary>
     /// Gets the angle selected by each absolute angular-mode displacement.
     /// </summary>
@@ -133,7 +102,7 @@ internal static partial class HevcIntraPredictor
         switch (mode)
         {
             case PlanarMode:
-                Predict<PlanarPredictionOperator>(
+                Predict<PlanarOperator>(
                     top,
                     left,
                     destination,
@@ -145,7 +114,7 @@ internal static partial class HevcIntraPredictor
                     scratch);
                 break;
             case DcMode:
-                Predict<DcPredictionOperator>(
+                Predict<DcOperator>(
                     top,
                     left,
                     destination,
@@ -157,7 +126,7 @@ internal static partial class HevcIntraPredictor
                     scratch);
                 break;
             default:
-                Predict<AngularPredictionOperator>(
+                Predict<AngularOperator>(
                     top,
                     left,
                     destination,
@@ -243,7 +212,7 @@ internal static partial class HevcIntraPredictor
         int bitDepth,
         bool filterPredictionEdges,
         Span<ushort> scratch)
-        where TOperator : struct, IHevcIntraPredictionOperator<TOperator>
+        where TOperator : struct, IHevcIntraPredictionOperator
         => TOperator.Predict(
             top,
             left,

src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine16Operator.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines the sixteen-point inverse discrete cosine transform operator.
+/// </content>
+internal static partial class HevcInverseTransformer
+{
+    /// <summary>
+    /// Implements the sixteen-point inverse discrete cosine transform.
+    /// </summary>
+    private readonly struct DiscreteCosine16Operator : IHevcInverseTransformOperator
+    {
+        /// <inheritdoc/>
+        public static int Size => 16;
+
+        /// <inheritdoc/>
+        public static bool UsesButterfly => true;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int GetCoefficient(int frequency, int position)
+        {
+            if (frequency == 0)
+            {
+                return 64;
+            }
+
+            int angle = ((2 * position) + 1) * frequency * 2;
+            angle &= 127;
+            if (angle > 64)
+            {
+                angle = 128 - angle;
+            }
+
+            // The second quadrant reuses the first-quadrant magnitude with a negative sign.
+            if (angle > 32)
+            {
+                return -DiscreteCosineMagnitudes[64 - angle];
+            }
+
+            return DiscreteCosineMagnitudes[angle];
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine32Operator.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines the thirty-two-point inverse discrete cosine transform operator.
+/// </content>
+internal static partial class HevcInverseTransformer
+{
+    /// <summary>
+    /// Implements the thirty-two-point inverse discrete cosine transform.
+    /// </summary>
+    private readonly struct DiscreteCosine32Operator : IHevcInverseTransformOperator
+    {
+        /// <inheritdoc/>
+        public static int Size => 32;
+
+        /// <inheritdoc/>
+        public static bool UsesButterfly => true;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int GetCoefficient(int frequency, int position)
+        {
+            if (frequency == 0)
+            {
+                return 64;
+            }
+
+            int angle = ((2 * position) + 1) * frequency * 1;
+            angle &= 127;
+            if (angle > 64)
+            {
+                angle = 128 - angle;
+            }
+
+            // The second quadrant reuses the first-quadrant magnitude with a negative sign.
+            if (angle > 32)
+            {
+                return -DiscreteCosineMagnitudes[64 - angle];
+            }
+
+            return DiscreteCosineMagnitudes[angle];
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine4Operator.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines the four-point inverse discrete cosine transform operator.
+/// </content>
+internal static partial class HevcInverseTransformer
+{
+    /// <summary>
+    /// Implements the four-point inverse discrete cosine transform.
+    /// </summary>
+    private readonly struct DiscreteCosine4Operator : IHevcInverseTransformOperator
+    {
+        /// <inheritdoc/>
+        public static int Size => 4;
+
+        /// <inheritdoc/>
+        public static bool UsesButterfly => true;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int GetCoefficient(int frequency, int position)
+        {
+            if (frequency == 0)
+            {
+                return 64;
+            }
+
+            int angle = ((2 * position) + 1) * frequency * 8;
+            angle &= 127;
+            if (angle > 64)
+            {
+                angle = 128 - angle;
+            }
+
+            // The second quadrant reuses the first-quadrant magnitude with a negative sign.
+            if (angle > 32)
+            {
+                return -DiscreteCosineMagnitudes[64 - angle];
+            }
+
+            return DiscreteCosineMagnitudes[angle];
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine8Operator.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines the eight-point inverse discrete cosine transform operator.
+/// </content>
+internal static partial class HevcInverseTransformer
+{
+    /// <summary>
+    /// Implements the eight-point inverse discrete cosine transform.
+    /// </summary>
+    private readonly struct DiscreteCosine8Operator : IHevcInverseTransformOperator
+    {
+        /// <inheritdoc/>
+        public static int Size => 8;
+
+        /// <inheritdoc/>
+        public static bool UsesButterfly => true;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int GetCoefficient(int frequency, int position)
+        {
+            if (frequency == 0)
+            {
+                return 64;
+            }
+
+            int angle = ((2 * position) + 1) * frequency * 4;
+            angle &= 127;
+            if (angle > 64)
+            {
+                angle = 128 - angle;
+            }
+
+            // The second quadrant reuses the first-quadrant magnitude with a negative sign.
+            if (angle > 32)
+            {
+                return -DiscreteCosineMagnitudes[64 - angle];
+            }
+
+            return DiscreteCosineMagnitudes[angle];
+        }
+    }
+}

src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteSine4Operator.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
+
+/// <content>
+/// Defines the four-point inverse discrete sine transform operator.
+/// </content>
+internal static partial class HevcInverseTransformer
+{
+    /// <summary>
+    /// Implements the four-point inverse discrete sine transform.
+    /// </summary>
+    private readonly struct DiscreteSine4Operator : IHevcInverseTransformOperator
+    {
+        /// <summary>
+        /// Gets the inverse-DST matrix in frequency-major order.
+        /// </summary>
+        private static ReadOnlySpan<sbyte> Coefficients =>
+        [
+            29, 55, 74, 84,
+            74, 74, 0, -74,
+            84, -29, -74, 55,
+            55, -84, 74, -29
+        ];
+
+        /// <inheritdoc/>
+        public static int Size => 4;
+
+        /// <inheritdoc/>
+        public static bool UsesButterfly => false;
+
+        /// <inheritdoc/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int GetCoefficient(int frequency, int position)
+        {
+            return Coefficients[(frequency * Size) + position];
+        }
+    }
+}