diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.ByteOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.ByteOutputOperator.cs new file mode 100644 index 000000000..3066c6841 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.ByteOutputOperator.cs @@ -0,0 +1,34 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Writes filtered samples to eight-bit plane storage. + /// + private readonly struct ByteOutputOperator : IOutputOperator + { + /// + public static void StoreVector(ref byte destination, int offset, Vector128 value, int count) + { + Vector64 packed = Vector128.Narrow(value.AsUInt16(), Vector128.Zero).GetLower(); + ref byte output = ref Unsafe.Add(ref destination, offset); + if (count == 8) + { + packed.StoreUnsafe(ref output); + } + else + { + Unsafe.WriteUnaligned(ref output, packed.AsUInt32().ToScalar()); + } + } + + /// + public static void StoreScalar(ref byte destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (byte)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.CopyFilterOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.CopyFilterOperator.cs new file mode 100644 index 000000000..8a28ad418 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.CopyFilterOperator.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Disables both tap groups so the source block is copied unchanged. + /// + private readonly struct CopyFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => false; + + /// + public static bool EnableSecondary => false; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.Operator.cs new file mode 100644 index 000000000..df9e5b060 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.Operator.cs @@ -0,0 +1,50 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Defines storage-specific writes for one filtered row. + /// + /// The destination sample storage type. + private interface IOutputOperator + where TSample : unmanaged + { + /// + /// Stores four or eight filtered samples from the low vector lanes. + /// + /// The first element in the destination plane. + /// The offset of the first sample to write. + /// The filtered samples in the low lanes. + /// The number of valid lanes. + public static abstract void StoreVector(ref TSample destination, int offset, Vector128 value, int count); + + /// + /// Stores one filtered sample. + /// + /// The first element in the destination plane. + /// The offset of the sample to write. + /// The filtered sample. + public static abstract void StoreScalar(ref TSample destination, int offset, int value); + } + + /// + /// Defines which groups of directional taps participate in one closed filter kernel. + /// + private interface IFilterOperator + { + /// + /// Gets a value indicating whether the primary directional taps are enabled. + /// + public static abstract bool EnablePrimary { get; } + + /// + /// Gets a value indicating whether the secondary off-axis taps are enabled. + /// + public static abstract bool EnableSecondary { get; } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryAndSecondaryFilterOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryAndSecondaryFilterOperator.cs new file mode 100644 index 000000000..2bfc98a7d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryAndSecondaryFilterOperator.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Enables both directional tap groups and their combined clipping rule. + /// + private readonly struct PrimaryAndSecondaryFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => true; + + /// + public static bool EnableSecondary => true; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryFilterOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryFilterOperator.cs new file mode 100644 index 000000000..091835f8e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.PrimaryFilterOperator.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Enables only the primary directional taps. + /// + private readonly struct PrimaryFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => true; + + /// + public static bool EnableSecondary => false; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.SecondaryFilterOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.SecondaryFilterOperator.cs new file mode 100644 index 000000000..0a1521c02 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.SecondaryFilterOperator.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Enables only the secondary off-axis taps. + /// + private readonly struct SecondaryFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => false; + + /// + public static bool EnableSecondary => true; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.UInt16OutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.UInt16OutputOperator.cs new file mode 100644 index 000000000..df80b2031 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.UInt16OutputOperator.cs @@ -0,0 +1,34 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +internal static partial class Av1CdefFilter +{ + /// + /// Writes filtered samples to 16-bit plane storage. + /// + private readonly struct UInt16OutputOperator : IOutputOperator + { + /// + public static void StoreVector(ref ushort destination, int offset, Vector128 value, int count) + { + ref ushort output = ref Unsafe.Add(ref destination, offset); + if (count == 8) + { + value.AsUInt16().StoreUnsafe(ref output); + } + else + { + ref byte outputBytes = ref Unsafe.As(ref output); + Unsafe.WriteUnaligned(ref outputBytes, value.AsUInt64().GetLower().ToScalar()); + } + } + + /// + public static void StoreScalar(ref ushort destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (ushort)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs index 5e04a01f6..fe39e14c0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs @@ -18,54 +18,13 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; /// one eight-by-eight block per 128-bit lane so AVX2 can evaluate two independent blocks together. Scalar kernels retain /// the same constrain, clipping, and tie-breaking rules for unsupported hardware and partial edge blocks. /// -internal static class Av1CdefFilter +internal static partial class Av1CdefFilter { /// /// The sample value used in the bordered source plane for neighbors outside the coded frame. /// public const ushort VeryLarge = 0x4000; - /// - /// Defines storage-specific writes for one filtered row. - /// - /// The destination sample storage type. - private interface IOutputOperator - where TSample : unmanaged - { - /// - /// Stores four or eight filtered samples from the low vector lanes. - /// - /// The first element in the destination plane. - /// The offset of the first sample to write. - /// The filtered samples in the low lanes. - /// The number of valid lanes. - public static abstract void StoreVector(ref TSample destination, int offset, Vector128 value, int count); - - /// - /// Stores one filtered sample. - /// - /// The first element in the destination plane. - /// The offset of the sample to write. - /// The filtered sample. - public static abstract void StoreScalar(ref TSample destination, int offset, int value); - } - - /// - /// Defines which groups of directional taps participate in one closed filter kernel. - /// - private interface IFilterOperator - { - /// - /// Gets a value indicating whether the primary directional taps are enabled. - /// - public static abstract bool EnablePrimary { get; } - - /// - /// Gets a value indicating whether the secondary off-axis taps are enabled. - /// - public static abstract bool EnableSecondary { get; } - } - /// /// Copies an eight-bit sample rectangle into the 16-bit CDEF working plane. /// @@ -1732,100 +1691,4 @@ internal static class Av1CdefFilter 6 => tap == 0 ? stride : 2 * stride, _ => tap == 0 ? stride : (2 * stride) - 1 }; - - /// - /// Enables both directional tap groups and their combined clipping rule. - /// - private readonly struct PrimaryAndSecondaryFilterOperator : IFilterOperator - { - /// - public static bool EnablePrimary => true; - - /// - public static bool EnableSecondary => true; - } - - /// - /// Enables only the primary directional taps. - /// - private readonly struct PrimaryFilterOperator : IFilterOperator - { - /// - public static bool EnablePrimary => true; - - /// - public static bool EnableSecondary => false; - } - - /// - /// Enables only the secondary off-axis taps. - /// - private readonly struct SecondaryFilterOperator : IFilterOperator - { - /// - public static bool EnablePrimary => false; - - /// - public static bool EnableSecondary => true; - } - - /// - /// Disables both tap groups so the source block is copied unchanged. - /// - private readonly struct CopyFilterOperator : IFilterOperator - { - /// - public static bool EnablePrimary => false; - - /// - public static bool EnableSecondary => false; - } - - /// - /// Writes filtered samples to eight-bit plane storage. - /// - private readonly struct ByteOutputOperator : IOutputOperator - { - /// - public static void StoreVector(ref byte destination, int offset, Vector128 value, int count) - { - Vector64 packed = Vector128.Narrow(value.AsUInt16(), Vector128.Zero).GetLower(); - ref byte output = ref Unsafe.Add(ref destination, offset); - if (count == 8) - { - packed.StoreUnsafe(ref output); - } - else - { - Unsafe.WriteUnaligned(ref output, packed.AsUInt32().ToScalar()); - } - } - - /// - public static void StoreScalar(ref byte destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (byte)value; - } - - /// - /// Writes filtered samples to 16-bit plane storage. - /// - private readonly struct UInt16OutputOperator : IOutputOperator - { - /// - public static void StoreVector(ref ushort destination, int offset, Vector128 value, int count) - { - ref ushort output = ref Unsafe.Add(ref destination, offset); - if (count == 8) - { - value.AsUInt16().StoreUnsafe(ref output); - } - else - { - ref byte outputBytes = ref Unsafe.As(ref output); - Unsafe.WriteUnaligned(ref outputBytes, value.AsUInt64().GetLower().ToScalar()); - } - } - - /// - public static void StoreScalar(ref ushort destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (ushort)value; - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs index 48224ef0c..847204f1b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs @@ -271,10 +271,10 @@ internal static class Av1FilmGrainNoise for (; column <= vectorEnd; column += Vector256.Count) { ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column); - Vector256 source = Av1FilmGrainSampleOperator.Load8(ref destination); + Vector256 source = Av1FilmGrainSampleOperations.Load8(ref destination); Vector256 grainValues = Vector256.LoadUnsafe(ref grainBase, (nuint)(grainRowOffset + column)); Vector256 result = AddNoise(source, grainValues, scaling, bitDepth, roundingOffset, scalingShift, minimum, maximum); - Av1FilmGrainSampleOperator.Store8(ref destination, result); + Av1FilmGrainSampleOperations.Store8(ref destination, result); } ApplyLumaScalar( @@ -326,10 +326,10 @@ internal static class Av1FilmGrainNoise for (; column <= vectorEnd; column += Vector128.Count) { ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column); - Vector128 source = Av1FilmGrainSampleOperator.Load4(ref destination); + Vector128 source = Av1FilmGrainSampleOperations.Load4(ref destination); Vector128 grainValues = Vector128.LoadUnsafe(ref grainBase, (nuint)(grainRowOffset + column)); Vector128 result = AddNoise(source, grainValues, scaling, bitDepth, roundingOffset, scalingShift, minimum, maximum); - Av1FilmGrainSampleOperator.Store4(ref destination, result); + Av1FilmGrainSampleOperations.Store4(ref destination, result); } ApplyLumaScalar( @@ -375,10 +375,10 @@ internal static class Av1FilmGrainNoise for (int column = 0; column < width; column++) { ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column); - int source = Av1FilmGrainSampleOperator.Load(ref destination); + int source = Av1FilmGrainSampleOperations.Load(ref destination); int scale = ScaleLookup(scaling, source, bitDepth); int value = source + (((scale * Unsafe.Add(ref grainBase, grainRowOffset + column)) + roundingOffset) >> scalingShift); - Av1FilmGrainSampleOperator.Store(ref destination, Av1Math.Clamp(value, minimum, maximum)); + Av1FilmGrainSampleOperations.Store(ref destination, Av1Math.Clamp(value, minimum, maximum)); } } } @@ -577,11 +577,11 @@ internal static class Av1FilmGrainNoise for (; column <= vectorEnd; column += Vector256.Count) { ref TSample lumaSource = ref Unsafe.Add(ref lumaRow, column << subsamplingX); - Vector256 averageLuma = Av1FilmGrainSampleOperator.LoadChromaLuma8(ref lumaSource, subsamplingX); + Vector256 averageLuma = Av1FilmGrainSampleOperations.LoadChromaLuma8(ref lumaSource, subsamplingX); if (applyCb) { ref TSample destination = ref Unsafe.Add(ref cbBase, chromaRowOffset + column); - Vector256 source = Av1FilmGrainSampleOperator.Load8(ref destination); + Vector256 source = Av1FilmGrainSampleOperations.Load8(ref destination); Vector256 scalingIndex = ((averageLuma * cbLumaMultiplier) + (source * cbMultiplier)) >> 6; scalingIndex = Vector256.Min(Vector256.Max(scalingIndex + Vector256.Create(cbOffset), zero), maximumIndex); Vector256 grainValues = Vector256.LoadUnsafe(ref cbGrainBase, (nuint)(grainRowOffset + column)); @@ -596,13 +596,13 @@ internal static class Av1FilmGrainNoise minimum, maximum); - Av1FilmGrainSampleOperator.Store8(ref destination, result); + Av1FilmGrainSampleOperations.Store8(ref destination, result); } if (applyCr) { ref TSample destination = ref Unsafe.Add(ref crBase, chromaRowOffset + column); - Vector256 source = Av1FilmGrainSampleOperator.Load8(ref destination); + Vector256 source = Av1FilmGrainSampleOperations.Load8(ref destination); Vector256 scalingIndex = ((averageLuma * crLumaMultiplier) + (source * crMultiplier)) >> 6; scalingIndex = Vector256.Min(Vector256.Max(scalingIndex + Vector256.Create(crOffset), zero), maximumIndex); Vector256 grainValues = Vector256.LoadUnsafe(ref crGrainBase, (nuint)(grainRowOffset + column)); @@ -617,7 +617,7 @@ internal static class Av1FilmGrainNoise minimum, maximum); - Av1FilmGrainSampleOperator.Store8(ref destination, result); + Av1FilmGrainSampleOperations.Store8(ref destination, result); } } @@ -708,11 +708,11 @@ internal static class Av1FilmGrainNoise for (; column <= vectorEnd; column += Vector128.Count) { ref TSample lumaSource = ref Unsafe.Add(ref lumaRow, column << subsamplingX); - Vector128 averageLuma = Av1FilmGrainSampleOperator.LoadChromaLuma4(ref lumaSource, subsamplingX); + Vector128 averageLuma = Av1FilmGrainSampleOperations.LoadChromaLuma4(ref lumaSource, subsamplingX); if (applyCb) { ref TSample destination = ref Unsafe.Add(ref cbBase, chromaRowOffset + column); - Vector128 source = Av1FilmGrainSampleOperator.Load4(ref destination); + Vector128 source = Av1FilmGrainSampleOperations.Load4(ref destination); Vector128 scalingIndex = ((averageLuma * cbLumaMultiplier) + (source * cbMultiplier)) >> 6; scalingIndex = Vector128.Min(Vector128.Max(scalingIndex + Vector128.Create(cbOffset), zero), maximumIndex); Vector128 grainValues = Vector128.LoadUnsafe(ref cbGrainBase, (nuint)(grainRowOffset + column)); @@ -727,13 +727,13 @@ internal static class Av1FilmGrainNoise minimum, maximum); - Av1FilmGrainSampleOperator.Store4(ref destination, result); + Av1FilmGrainSampleOperations.Store4(ref destination, result); } if (applyCr) { ref TSample destination = ref Unsafe.Add(ref crBase, chromaRowOffset + column); - Vector128 source = Av1FilmGrainSampleOperator.Load4(ref destination); + Vector128 source = Av1FilmGrainSampleOperations.Load4(ref destination); Vector128 scalingIndex = ((averageLuma * crLumaMultiplier) + (source * crMultiplier)) >> 6; scalingIndex = Vector128.Min(Vector128.Max(scalingIndex + Vector128.Create(crOffset), zero), maximumIndex); Vector128 grainValues = Vector128.LoadUnsafe(ref crGrainBase, (nuint)(grainRowOffset + column)); @@ -748,7 +748,7 @@ internal static class Av1FilmGrainNoise minimum, maximum); - Av1FilmGrainSampleOperator.Store4(ref destination, result); + Av1FilmGrainSampleOperations.Store4(ref destination, result); } } @@ -831,10 +831,10 @@ internal static class Av1FilmGrainNoise for (int column = 0; column < width; column++) { int lumaOffset = lumaRowOffset + (column << subsamplingX); - int averageLuma = Av1FilmGrainSampleOperator.Load(ref Unsafe.Add(ref lumaBase, lumaOffset)); + int averageLuma = Av1FilmGrainSampleOperations.Load(ref Unsafe.Add(ref lumaBase, lumaOffset)); if (subsamplingX != 0) { - averageLuma = (averageLuma + Av1FilmGrainSampleOperator.Load(ref Unsafe.Add(ref lumaBase, lumaOffset + 1)) + 1) >> 1; + averageLuma = (averageLuma + Av1FilmGrainSampleOperations.Load(ref Unsafe.Add(ref lumaBase, lumaOffset + 1)) + 1) >> 1; } int chromaOffset = chromaRowOffset + column; @@ -842,7 +842,7 @@ internal static class Av1FilmGrainNoise if (applyCb) { ref TSample destination = ref Unsafe.Add(ref cbBase, chromaOffset); - int source = Av1FilmGrainSampleOperator.Load(ref destination); + int source = Av1FilmGrainSampleOperations.Load(ref destination); int scalingIndex = Av1Math.Clamp( (((averageLuma * cbLumaMultiplier) + (source * cbMultiplier)) >> 6) + cbOffset, 0, @@ -850,13 +850,13 @@ internal static class Av1FilmGrainNoise int scale = ScaleLookup(scalingCb, scalingIndex, bitDepth); int value = source + (((scale * Unsafe.Add(ref cbGrainBase, grainOffset)) + roundingOffset) >> scalingShift); - Av1FilmGrainSampleOperator.Store(ref destination, Av1Math.Clamp(value, minimum, maximum)); + Av1FilmGrainSampleOperations.Store(ref destination, Av1Math.Clamp(value, minimum, maximum)); } if (applyCr) { ref TSample destination = ref Unsafe.Add(ref crBase, chromaOffset); - int source = Av1FilmGrainSampleOperator.Load(ref destination); + int source = Av1FilmGrainSampleOperations.Load(ref destination); int scalingIndex = Av1Math.Clamp( (((averageLuma * crLumaMultiplier) + (source * crMultiplier)) >> 6) + crOffset, 0, @@ -864,7 +864,7 @@ internal static class Av1FilmGrainNoise int scale = ScaleLookup(scalingCr, scalingIndex, bitDepth); int value = source + (((scale * Unsafe.Add(ref crGrainBase, grainOffset)) + roundingOffset) >> scalingShift); - Av1FilmGrainSampleOperator.Store(ref destination, Av1Math.Clamp(value, minimum, maximum)); + Av1FilmGrainSampleOperations.Store(ref destination, Av1Math.Clamp(value, minimum, maximum)); } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperations.cs similarity index 99% rename from src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperator.cs rename to src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperations.cs index ad648a7c3..eb855a710 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperations.cs @@ -18,7 +18,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain; /// row loops. All arithmetic uses signed 32-bit lanes; decoded samples are nonnegative and at most twelve bits, making /// the intermediate signed 16-bit views safe wherever pairwise operations require them. /// -internal readonly struct Av1FilmGrainSampleOperator +internal readonly struct Av1FilmGrainSampleOperations where TSample : unmanaged { /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalByteEdgeOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalByteEdgeOperator.cs new file mode 100644 index 000000000..8871d36a4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalByteEdgeOperator.cs @@ -0,0 +1,45 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; + +internal static partial class Av1DeblockingFilter +{ + /// + /// Accesses four columns across a horizontal edge in eight-bit storage. + /// + private readonly struct HorizontalByteEdgeOperator : IEdgeOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 LoadVector(ref byte samples, int q0Offset, int stride, int distance) + { + ref byte source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride)); + uint packed = Unsafe.ReadUnaligned(ref source); + Vector128 widened = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + return Vector128.WidenLower(widened).AsInt32(); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128 value) + { + Vector128 narrowed16 = Vector128.Narrow(value.AsUInt32(), Vector128.Zero); + Vector128 narrowed8 = Vector128.Narrow(narrowed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref samples, q0Offset + (distance * stride)), narrowed8.AsUInt32().ToScalar()); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index) + => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value) + => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (byte)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalUInt16EdgeOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalUInt16EdgeOperator.cs new file mode 100644 index 000000000..115b3bae5 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalUInt16EdgeOperator.cs @@ -0,0 +1,44 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; + +internal static partial class Av1DeblockingFilter +{ + /// + /// Accesses four columns across a horizontal edge in 16-bit storage. + /// + private readonly struct HorizontalUInt16EdgeOperator : IEdgeOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 LoadVector(ref ushort samples, int q0Offset, int stride, int distance) + { + ref ushort source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride)); + ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref source)); + return Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsUInt16()).AsInt32(); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128 value) + { + Vector64 narrowed = Vector128.Narrow(value, Vector128.Zero).AsUInt16().GetLower(); + ref byte destination = ref Unsafe.As(ref Unsafe.Add(ref samples, q0Offset + (distance * stride))); + Unsafe.WriteUnaligned(ref destination, narrowed.AsUInt64().ToScalar()); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index) + => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value) + => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (ushort)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.Operator.cs new file mode 100644 index 000000000..18f949b04 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.Operator.cs @@ -0,0 +1,59 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; + +internal static partial class Av1DeblockingFilter +{ + /// + /// Defines orientation- and storage-specific access to the four samples running along one edge segment. + /// + /// The reconstructed sample storage type. + private interface IEdgeOperator + where TSample : unmanaged + { + /// + /// Loads four samples at one signed distance across the edge. + /// + /// The first element in the plane storage. + /// The offset of the first Q-side sample. + /// The number of samples between adjacent rows. + /// The signed sample distance from Q0. + /// The widened samples ordered along the edge. + public static abstract Vector128 LoadVector(ref TSample samples, int q0Offset, int stride, int distance); + + /// + /// Stores four samples at one signed distance across the edge. + /// + /// The first element in the plane storage. + /// The offset of the first Q-side sample. + /// The number of samples between adjacent rows. + /// The signed sample distance from Q0. + /// The widened samples ordered along the edge. + public static abstract void StoreVector(ref TSample samples, int q0Offset, int stride, int distance, Vector128 value); + + /// + /// Loads one sample at a signed distance across and an offset along the edge. + /// + /// The first element in the plane storage. + /// The offset of the first Q-side sample. + /// The number of samples between adjacent rows. + /// The signed sample distance from Q0. + /// The sample offset along the edge. + /// The selected sample. + public static abstract int LoadScalar(ref TSample samples, int q0Offset, int stride, int distance, int index); + + /// + /// Stores one sample at a signed distance across and an offset along the edge. + /// + /// The first element in the plane storage. + /// The offset of the first Q-side sample. + /// The number of samples between adjacent rows. + /// The signed sample distance from Q0. + /// The sample offset along the edge. + /// The filtered sample. + public static abstract void StoreScalar(ref TSample samples, int q0Offset, int stride, int distance, int index, int value); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalByteEdgeOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalByteEdgeOperator.cs new file mode 100644 index 000000000..cd491a755 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalByteEdgeOperator.cs @@ -0,0 +1,45 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; + +internal static partial class Av1DeblockingFilter +{ + /// + /// Accesses four rows across a vertical edge in eight-bit storage. + /// + private readonly struct VerticalByteEdgeOperator : IEdgeOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 LoadVector(ref byte samples, int q0Offset, int stride, int distance) + => Vector128.Create( + (int)Unsafe.Add(ref samples, q0Offset + distance), + Unsafe.Add(ref samples, q0Offset + stride + distance), + Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance), + Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance)); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128 value) + { + Unsafe.Add(ref samples, q0Offset + distance) = (byte)value.GetElement(0); + Unsafe.Add(ref samples, q0Offset + stride + distance) = (byte)value.GetElement(1); + Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (byte)value.GetElement(2); + Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (byte)value.GetElement(3); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index) + => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value) + => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (byte)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalUInt16EdgeOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalUInt16EdgeOperator.cs new file mode 100644 index 000000000..707c4226c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalUInt16EdgeOperator.cs @@ -0,0 +1,45 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; + +internal static partial class Av1DeblockingFilter +{ + /// + /// Accesses four rows across a vertical edge in 16-bit storage. + /// + private readonly struct VerticalUInt16EdgeOperator : IEdgeOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 LoadVector(ref ushort samples, int q0Offset, int stride, int distance) + => Vector128.Create( + (int)Unsafe.Add(ref samples, q0Offset + distance), + Unsafe.Add(ref samples, q0Offset + stride + distance), + Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance), + Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance)); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128 value) + { + Unsafe.Add(ref samples, q0Offset + distance) = (ushort)value.GetElement(0); + Unsafe.Add(ref samples, q0Offset + stride + distance) = (ushort)value.GetElement(1); + Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (ushort)value.GetElement(2); + Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (ushort)value.GetElement(3); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index) + => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value) + => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (ushort)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs index 04e36a0d7..b74e1b452 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs @@ -16,58 +16,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; /// lane-wise. Conditional selection preserves unfiltered lanes while allowing four adjacent edge positions to share /// one kernel invocation. /// -internal static class Av1DeblockingFilter +internal static partial class Av1DeblockingFilter { - /// - /// Defines orientation- and storage-specific access to the four samples running along one edge segment. - /// - /// The reconstructed sample storage type. - private interface IEdgeOperator - where TSample : unmanaged - { - /// - /// Loads four samples at one signed distance across the edge. - /// - /// The first element in the plane storage. - /// The offset of the first Q-side sample. - /// The number of samples between adjacent rows. - /// The signed sample distance from Q0. - /// The widened samples ordered along the edge. - public static abstract Vector128 LoadVector(ref TSample samples, int q0Offset, int stride, int distance); - - /// - /// Stores four samples at one signed distance across the edge. - /// - /// The first element in the plane storage. - /// The offset of the first Q-side sample. - /// The number of samples between adjacent rows. - /// The signed sample distance from Q0. - /// The widened samples ordered along the edge. - public static abstract void StoreVector(ref TSample samples, int q0Offset, int stride, int distance, Vector128 value); - - /// - /// Loads one sample at a signed distance across and an offset along the edge. - /// - /// The first element in the plane storage. - /// The offset of the first Q-side sample. - /// The number of samples between adjacent rows. - /// The signed sample distance from Q0. - /// The sample offset along the edge. - /// The selected sample. - public static abstract int LoadScalar(ref TSample samples, int q0Offset, int stride, int distance, int index); - - /// - /// Stores one sample at a signed distance across and an offset along the edge. - /// - /// The first element in the plane storage. - /// The offset of the first Q-side sample. - /// The number of samples between adjacent rows. - /// The signed sample distance from Q0. - /// The sample offset along the edge. - /// The filtered sample. - public static abstract void StoreScalar(ref TSample samples, int q0Offset, int stride, int distance, int index, int value); - } - /// /// Filters four rows crossing one vertical boundary in eight-bit storage. /// @@ -943,143 +893,4 @@ internal static class Av1DeblockingFilter /// The rounded quotient. private static int RoundPowerOfTwo(int value, int bitCount) => (value + (1 << (bitCount - 1))) >> bitCount; - - /// - /// Accesses four rows across a vertical edge in eight-bit storage. - /// - private readonly struct VerticalByteEdgeOperator : IEdgeOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(ref byte samples, int q0Offset, int stride, int distance) - => Vector128.Create( - (int)Unsafe.Add(ref samples, q0Offset + distance), - Unsafe.Add(ref samples, q0Offset + stride + distance), - Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance), - Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance)); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128 value) - { - Unsafe.Add(ref samples, q0Offset + distance) = (byte)value.GetElement(0); - Unsafe.Add(ref samples, q0Offset + stride + distance) = (byte)value.GetElement(1); - Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (byte)value.GetElement(2); - Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (byte)value.GetElement(3); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index) - => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value) - => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (byte)value; - } - - /// - /// Accesses four columns across a horizontal edge in eight-bit storage. - /// - private readonly struct HorizontalByteEdgeOperator : IEdgeOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(ref byte samples, int q0Offset, int stride, int distance) - { - ref byte source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride)); - uint packed = Unsafe.ReadUnaligned(ref source); - Vector128 widened = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - return Vector128.WidenLower(widened).AsInt32(); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreVector(ref byte samples, int q0Offset, int stride, int distance, Vector128 value) - { - Vector128 narrowed16 = Vector128.Narrow(value.AsUInt32(), Vector128.Zero); - Vector128 narrowed8 = Vector128.Narrow(narrowed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref samples, q0Offset + (distance * stride)), narrowed8.AsUInt32().ToScalar()); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LoadScalar(ref byte samples, int q0Offset, int stride, int distance, int index) - => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreScalar(ref byte samples, int q0Offset, int stride, int distance, int index, int value) - => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (byte)value; - } - - /// - /// Accesses four rows across a vertical edge in 16-bit storage. - /// - private readonly struct VerticalUInt16EdgeOperator : IEdgeOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(ref ushort samples, int q0Offset, int stride, int distance) - => Vector128.Create( - (int)Unsafe.Add(ref samples, q0Offset + distance), - Unsafe.Add(ref samples, q0Offset + stride + distance), - Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance), - Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance)); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128 value) - { - Unsafe.Add(ref samples, q0Offset + distance) = (ushort)value.GetElement(0); - Unsafe.Add(ref samples, q0Offset + stride + distance) = (ushort)value.GetElement(1); - Unsafe.Add(ref samples, q0Offset + (2 * stride) + distance) = (ushort)value.GetElement(2); - Unsafe.Add(ref samples, q0Offset + (3 * stride) + distance) = (ushort)value.GetElement(3); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index) - => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value) - => Unsafe.Add(ref samples, q0Offset + (index * stride) + distance) = (ushort)value; - } - - /// - /// Accesses four columns across a horizontal edge in 16-bit storage. - /// - private readonly struct HorizontalUInt16EdgeOperator : IEdgeOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(ref ushort samples, int q0Offset, int stride, int distance) - { - ref ushort source = ref Unsafe.Add(ref samples, q0Offset + (distance * stride)); - ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref source)); - return Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsUInt16()).AsInt32(); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreVector(ref ushort samples, int q0Offset, int stride, int distance, Vector128 value) - { - Vector64 narrowed = Vector128.Narrow(value, Vector128.Zero).AsUInt16().GetLower(); - ref byte destination = ref Unsafe.As(ref Unsafe.Add(ref samples, q0Offset + (distance * stride))); - Unsafe.WriteUnaligned(ref destination, narrowed.AsUInt64().ToScalar()); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LoadScalar(ref ushort samples, int q0Offset, int stride, int distance, int index) - => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreScalar(ref ushort samples, int q0Offset, int stride, int distance, int index, int value) - => Unsafe.Add(ref samples, q0Offset + (distance * stride) + index) = (ushort)value; - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs index 650d6a21b..3aaa2a358 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs @@ -18,7 +18,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// Eight-bit blocks use saturating 16-bit stages where their normative ranges permit it; high-bit-depth and scalar /// fallback paths retain 32-bit stages. Both representations produce the same row-major coefficient contract. /// -internal static class Av1ForwardTransformer +internal static partial class Av1ForwardTransformer { /// /// Resolves and applies the configured two-dimensional AV1 forward transform. @@ -65,40 +65,40 @@ internal static class Av1ForwardTransformer switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeColumn} column transform is not valid for {config.TransformSize}."); @@ -127,40 +127,40 @@ internal static class Av1ForwardTransformer switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeRow} row transform is not valid for {config.TransformSize}."); @@ -545,7 +545,7 @@ internal static class Av1ForwardTransformer /// /// Applies one packed transform axis using the widest efficient lane count available for the block. /// - /// The transform operator applied to each independent axis. + /// The semantic transform operator. /// The packed transform block. /// The number of independent axes. /// The number of packed values between input positions. @@ -563,23 +563,23 @@ internal static class Av1ForwardTransformer { if (Avx512BW.IsSupported && transformCount >= Vector512.Count) { - TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformPackedVector512(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } if (Avx2.IsSupported && transformCount >= Vector256.Count) { - TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformPackedVector256(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } - TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformPackedVector128(buffer, transformCount, inputStride, outputStride, cosBit, workspace); } /// - /// Applies one signed thirty-two-bit transform axis using the widest efficient lane count available for the block. + /// Applies one expanded transform axis using the widest efficient lane count available for the block. /// - /// The transform operator applied to each independent axis. + /// The semantic transform operator. /// The expanded transform block. /// The number of independent axes. /// The number of expanded values between input positions. @@ -597,66 +597,218 @@ internal static class Av1ForwardTransformer { if (Vector512.IsHardwareAccelerated && transformCount >= Vector512.Count) { - TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformExpandedVector512(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } if (Vector256.IsHardwareAccelerated && transformCount >= Vector256.Count) { - TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformExpandedVector256(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } if (Vector128.IsHardwareAccelerated && transformCount >= Vector128.Count) { - TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformExpandedVector128(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } - TransformAxis(buffer, transformCount, inputStride, outputStride, cosBit, workspace); + TransformExpandedScalar(buffer, transformCount, inputStride, outputStride, cosBit, workspace); } /// - /// Applies one transform stage network to independent axes held in scalar or SIMD lanes. + /// Applies a packed transform to thirty-two independent axes. /// - /// The transform operator applied to each independent axis. - /// The scalar storage element. - /// The scalar or SIMD value containing independent transform axes. - /// The transform block. - /// The number of independent axes. - /// The number of storage elements between input positions. - /// The number of storage elements between output positions. - /// The fixed-point precision of the cosine constants. - /// The reusable transform-stage workspace. - private static void TransformAxis( - Span buffer, + private static void TransformPackedVector512( + Span buffer, + int transformCount, + int inputStride, + int outputStride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + ref Av1TransformVector> buffer0 = + ref Unsafe.As>>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector> buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref short source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(short); + nint outputByteStride = outputStride * sizeof(short); + + for (int batch = 0; batch < transformCount; batch += Vector512.Count) + { + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); + + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + } + } + + /// + /// Applies a packed transform to sixteen independent axes. + /// + private static void TransformPackedVector256( + Span buffer, + int transformCount, + int inputStride, + int outputStride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + ref Av1TransformVector> buffer0 = + ref Unsafe.As>>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector> buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref short source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(short); + nint outputByteStride = outputStride * sizeof(short); + + for (int batch = 0; batch < transformCount; batch += Vector256.Count) + { + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); + + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + } + } + + /// + /// Applies a packed transform to eight independent axes. + /// + private static void TransformPackedVector128( + Span buffer, + int transformCount, + int inputStride, + int outputStride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + ref Av1TransformVector> buffer0 = + ref Unsafe.As>>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector> buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref short source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(short); + nint outputByteStride = outputStride * sizeof(short); + + for (int batch = 0; batch < transformCount; batch += Vector128.Count) + { + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); + + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + } + } + + /// + /// Applies an expanded transform to sixteen independent axes. + /// + private static void TransformExpandedVector512( + Span buffer, + int transformCount, + int inputStride, + int outputStride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + ref Av1TransformVector> buffer0 = + ref Unsafe.As>>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector> buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref int source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(int); + nint outputByteStride = outputStride * sizeof(int); + + for (int batch = 0; batch < transformCount; batch += Vector512.Count) + { + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); + + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + } + } + + /// + /// Applies an expanded transform to eight independent axes. + /// + private static void TransformExpandedVector256( + Span buffer, + int transformCount, + int inputStride, + int outputStride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + ref Av1TransformVector> buffer0 = + ref Unsafe.As>>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector> buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref int source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(int); + nint outputByteStride = outputStride * sizeof(int); + + for (int batch = 0; batch < transformCount; batch += Vector256.Count) + { + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); + + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + } + } + + /// + /// Applies an expanded transform to four independent axes. + /// + private static void TransformExpandedVector128( + Span buffer, + int transformCount, + int inputStride, + int outputStride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + ref Av1TransformVector> buffer0 = + ref Unsafe.As>>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector> buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref int source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(int); + nint outputByteStride = outputStride * sizeof(int); + + for (int batch = 0; batch < transformCount; batch += Vector128.Count) + { + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); + + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + } + } + + /// + /// Applies an expanded transform to one axis. + /// + private static void TransformExpandedScalar( + Span buffer, int transformCount, int inputStride, int outputStride, int cosBit, Span workspace) where TOperator : struct, IAv1ForwardTransform1dOperator - where TElement : unmanaged - where TValue : struct { - int vectorByteLength = Unsafe.SizeOf>(); - int laneCount = Unsafe.SizeOf() / Unsafe.SizeOf(); - ref byte workspaceBase = ref Unsafe.As(ref MemoryMarshal.GetReference(workspace)); - ref Av1TransformVector buffer0 = ref Unsafe.As>(ref workspaceBase); - ref Av1TransformVector buffer1 = - ref Unsafe.As>(ref Unsafe.Add(ref workspaceBase, vectorByteLength)); - - ref TElement sourceBase = ref MemoryMarshal.GetReference(buffer); - nint inputByteStride = inputStride * Unsafe.SizeOf(); - nint outputByteStride = outputStride * Unsafe.SizeOf(); - - // Each lane is an independent row or column. The operators load from and retire coefficients directly to - // the strided block, matching Highway's two-buffer stage network without a separate input/output copy pass. - for (int batch = 0; batch < transformCount; batch += laneCount) + ref Av1TransformVector buffer0 = + ref Unsafe.As>(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector buffer1 = + ref Unsafe.Add(ref buffer0, 1); + ref int source = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * sizeof(int); + nint outputByteStride = outputStride * sizeof(int); + + for (int batch = 0; batch < transformCount; batch++) { - ref byte values = ref Unsafe.As(ref Unsafe.Add(ref sourceBase, batch)); + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref source, batch)); - TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs index 23ddc7e2f..fd7d0518a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs @@ -4,7 +4,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -17,7 +16,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// cross-lane permutations. Reconstruction adds the final residuals to their matching prediction lanes before /// narrowing to the decoded sample depth. /// -internal static class Av1Inverse2dTransformer +internal static partial class Av1Inverse2dTransformer { /// /// Applies an inverse transform and adds its residual to high-bit-depth predicted samples. @@ -39,7 +38,7 @@ internal static class Av1Inverse2dTransformer ref Av1Transform2dFlipConfiguration config, Span workspace, int bitDepth) - => Transform2dAdd>( + => Transform2dAdd( input, outputForRead, strideForRead, @@ -67,7 +66,7 @@ internal static class Av1Inverse2dTransformer int strideForWrite, ref Av1Transform2dFlipConfiguration config, Span workspace) - => Transform2dAdd>( + => Transform2dAdd( input, outputForRead, strideForRead, @@ -90,68 +89,68 @@ internal static class Av1Inverse2dTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator { Guard.MustBeSizedAtLeast(workspace, Av1TransformWorkspace.GetRequiredLength(config.TransformSize), nameof(workspace)); switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct32: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct64: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity32: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; @@ -173,68 +172,68 @@ internal static class Av1Inverse2dTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator where TColumnOperator : struct, IAv1Transform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct32: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct64: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity32: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; @@ -256,7 +255,7 @@ internal static class Av1Inverse2dTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { @@ -308,7 +307,7 @@ internal static class Av1Inverse2dTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { @@ -464,7 +463,7 @@ internal static class Av1Inverse2dTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { @@ -596,7 +595,7 @@ internal static class Av1Inverse2dTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs deleted file mode 100644 index 08d035cf8..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Common.Helpers; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Reconstructs AV1 samples from predicted values and inverse-transform residuals. -/// -/// -/// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to -/// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width -/// stores. The closed specialization removes storage-type branches from hot loops. -/// -/// The decoded sample storage type. -internal readonly struct Av1InverseTransformOutputOperator : IAv1InverseTransformOutputOperator - where TSample : unmanaged -{ - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static TSample Add(TSample prediction, int residual, int bitDepth) - { - // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each - // closed transform so storage selection does not introduce a branch in the reconstruction loop. - if (typeof(TSample) == typeof(byte)) - { - byte value = (byte)Math.Clamp(Unsafe.As(ref prediction) + residual, byte.MinValue, byte.MaxValue); - return Unsafe.As(ref value); - } - - short result = (short)Math.Clamp(Unsafe.As(ref prediction) + residual, 0, (1 << bitDepth) - 1); - return Unsafe.As(ref result); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth) - { - if (typeof(TSample) == typeof(byte)) - { - // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach - // memory, which keeps reconstruction valid at a tightly packed row boundary. - ref byte source = ref Unsafe.As(ref prediction); - uint packed = Unsafe.ReadUnaligned(ref source); - Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32(); - Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue)); - Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt32().ToScalar()); - return; - } - - ref short highBitDepthSource = ref Unsafe.As(ref prediction); - ulong highBitDepthPacked = Unsafe.ReadUnaligned(ref Unsafe.As(ref highBitDepthSource)); - Vector128 highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16()); - Vector128 highBitDepthReconstructed = - Vector128.Clamp(highBitDepthPredicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1)); - - Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar()); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth) - { - if (typeof(TSample) == typeof(byte)) - { - // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store - // covers only those reconstructed samples and does not require destination padding. - ref byte source = ref Unsafe.As(ref prediction); - ulong packed = Unsafe.ReadUnaligned(ref source); - Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32(); - Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue)); - Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt64().ToScalar()); - return; - } - - ref short highBitDepthSource = ref Unsafe.As(ref prediction); - Vector256 highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource)); - Vector256 highBitDepthReconstructed = - Vector256.Clamp(highBitDepthPredicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1)); - - Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper()); - narrowed.StoreUnsafe(ref Unsafe.As(ref destination)); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.ByteOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.ByteOutputOperator.cs new file mode 100644 index 000000000..cec05507d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.ByteOutputOperator.cs @@ -0,0 +1,61 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines eight-bit inverse-transform reconstruction arithmetic. +/// +internal partial class Av1InverseTransformer +{ + /// + /// Reconstructs eight-bit samples from predicted values and inverse-transform residuals. + /// + internal readonly struct ByteOutputOperator : IAv1InverseTransformOutputOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Add(byte prediction, int residual, int bitDepth) + { + _ = bitDepth; + return (byte)Math.Clamp(prediction + residual, byte.MinValue, byte.MaxValue); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref byte prediction, ref byte destination, Vector128 residual, int bitDepth) + { + _ = bitDepth; + + // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach + // memory, which keeps reconstruction valid at a tightly packed row boundary. + uint packed = Unsafe.ReadUnaligned(ref prediction); + Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32(); + Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue)); + Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt32().ToScalar()); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref byte prediction, ref byte destination, Vector256 residual, int bitDepth) + { + _ = bitDepth; + + // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store + // covers only those reconstructed samples and does not require destination padding. + ulong packed = Unsafe.ReadUnaligned(ref prediction); + Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32(); + Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue)); + Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt64().ToScalar()); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.HighBitDepthOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.HighBitDepthOutputOperator.cs new file mode 100644 index 000000000..19eb1c4a6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.HighBitDepthOutputOperator.cs @@ -0,0 +1,50 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines high-bit-depth inverse-transform reconstruction arithmetic. +/// +internal partial class Av1InverseTransformer +{ + /// + /// Reconstructs high-bit-depth samples from predicted values and inverse-transform residuals. + /// + internal readonly struct HighBitDepthOutputOperator : IAv1InverseTransformOutputOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Add(short prediction, int residual, int bitDepth) + => (short)Math.Clamp(prediction + residual, 0, (1 << bitDepth) - 1); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref short prediction, ref short destination, Vector128 residual, int bitDepth) + { + ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref prediction)); + Vector128 predicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsInt16()); + Vector128 reconstructed = + Vector128.Clamp(predicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1)); + + Vector128 narrowed = Vector128.Narrow(reconstructed, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar()); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref short prediction, ref short destination, Vector256 residual, int bitDepth) + { + Vector256 predicted = Vector256_.Widen(Vector128.LoadUnsafe(ref prediction)); + Vector256 reconstructed = + Vector256.Clamp(predicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1)); + + Vector128 narrowed = Vector128.Narrow(reconstructed.GetLower(), reconstructed.GetUpper()); + narrowed.StoreUnsafe(ref destination); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs new file mode 100644 index 000000000..c0b7f9159 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs @@ -0,0 +1,47 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the inverse-transform reconstruction operator contract. +/// +internal partial class Av1InverseTransformer +{ + /// + /// Defines how inverse-transform residuals are added to decoded samples. + /// + /// The decoded sample storage type. + internal interface IAv1InverseTransformOutputOperator + where TSample : unmanaged + { + /// + /// Adds one residual to a predicted sample and clips the result to the coded bit depth. + /// + /// The predicted sample. + /// The inverse-transform residual. + /// The coded sample bit depth. + /// The reconstructed sample. + public static abstract TSample Add(TSample prediction, int residual, int bitDepth); + + /// + /// Adds four residuals to predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The four inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth); + + /// + /// Adds eight residuals to predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The eight inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs index 5ba9ca4c8..1c72acbc0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs @@ -6,7 +6,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// /// Reconstructs decoded AV1 transform coefficients into prediction sample buffers. /// -internal class Av1InverseTransformer +internal partial class Av1InverseTransformer { /// /// Reconstructs an eight-bit transform block in place by adding its inverse-transform residual. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs index 034e1d5a4..afe4c7ef9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs @@ -40,7 +40,7 @@ internal static class Av1InverseWalshHadamardTransformer int writeStride, int coefficientCount, Span workspace) - => TransformAdd>( + => TransformAdd( coefficients, readBuffer, readStride, @@ -70,7 +70,7 @@ internal static class Av1InverseWalshHadamardTransformer int coefficientCount, Span workspace, int bitDepth) - => TransformAdd>( + => TransformAdd( coefficients, readBuffer, readStride, @@ -93,7 +93,7 @@ internal static class Av1InverseWalshHadamardTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator { if (Vector128.IsHardwareAccelerated) { @@ -116,7 +116,7 @@ internal static class Av1InverseWalshHadamardTransformer int coefficientCount, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator { ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); Vector128 row0; @@ -172,7 +172,7 @@ internal static class Av1InverseWalshHadamardTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator { ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer); ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer); @@ -300,7 +300,7 @@ internal static class Av1InverseWalshHadamardTransformer Vector128 row3, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator { ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer); ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs deleted file mode 100644 index 3c354fd6c..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator. -/// -internal readonly struct Av1Adst16Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs deleted file mode 100644 index ec561c746..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the four-point AV1 forward asymmetric discrete sine transform operator. -/// -internal readonly struct Av1Adst4Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs deleted file mode 100644 index 1d7e0a424..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the eight-point AV1 forward asymmetric discrete sine transform operator. -/// -internal readonly struct Av1Adst8Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs deleted file mode 100644 index e1b447b15..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixteen-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct16Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs deleted file mode 100644 index 5749ab40e..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the thirty-two-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct32Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs deleted file mode 100644 index bef5993f3..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the four-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct4Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs deleted file mode 100644 index ca12aa88a..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixty-four-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct64Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs deleted file mode 100644 index 8fc637085..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the eight-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct8Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs deleted file mode 100644 index 9473e7663..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs +++ /dev/null @@ -1,314 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Implements the forward asymmetric discrete sine transform stage networks. -/// -internal static partial class Av1ForwardTransformOperations -{ - /// - /// Gets the fixed eight-point ADST coefficient permutation. - /// - private static ReadOnlySpan Adst8OutputOrder => [1, 6, 3, 4, 5, 2, 7, 0]; - - /// - /// Gets the first cosine index for each final sixteen-point ADST rotation. - /// - private static ReadOnlySpan Adst16FinalWeights => [2, 10, 18, 26, 34, 42, 50, 58]; - - /// - /// Gets the fixed sixteen-point ADST coefficient permutation. - /// - private static ReadOnlySpan Adst16OutputOrder => [1, 14, 3, 12, 5, 10, 7, 8, 9, 6, 11, 4, 13, 2, 15, 0]; - - /// - /// Applies the four-point forward asymmetric discrete sine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The unused first transform-stage buffer. - /// The unused second transform-stage buffer. - /// The fixed-point precision of the sine constants. - public static void Adst4( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - _ = buffer0; - _ = buffer1; - - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - TValue input0 = Load(ref values, inputStride, 0); - TValue input1 = Load(ref values, inputStride, 1); - TValue input2 = Load(ref values, inputStride, 2); - TValue input3 = Load(ref values, inputStride, 3); - TValue input01 = Av1ForwardTransformArithmetic.Add(input0, input1); - - // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. - TValue output0 = Av1ForwardTransformArithmetic.MultiplyAddRound( - sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); - - TValue output1 = Av1ForwardTransformArithmetic.MultiplyAddRound( - sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); - - TValue output2 = Av1ForwardTransformArithmetic.MultiplyAddRound( - sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); - - // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. - TValue output3 = Av1ForwardTransformArithmetic.MultiplyAddRound( - sinpi[4] - sinpi[1], - input0, - -sinpi[1] - sinpi[2], - input1, - sinpi[3], - input2, - sinpi[2] - sinpi[4], - input3, - cosBit, - in rounding); - - Store(ref values, outputStride, 0, output0); - Store(ref values, outputStride, 1, output1); - Store(ref values, outputStride, 2, output2); - Store(ref values, outputStride, 3, output3); - } - - /// - /// Applies the eight-point forward asymmetric discrete sine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Adst8( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - // Stage 1 applies the ADST permutation and signs while the source block is still read-only. - buffer0[0] = Load(ref values, inputStride, 0); - buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); - buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); - buffer0[3] = Load(ref values, inputStride, 4); - buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); - buffer0[5] = Load(ref values, inputStride, 6); - buffer0[6] = Load(ref values, inputStride, 2); - buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); - - // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. - buffer1[0] = buffer0[0]; - buffer1[1] = buffer0[1]; - Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); - buffer1[4] = buffer0[4]; - buffer1[5] = buffer0[5]; - Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); - - // Stage 3 combines the rotated and copied pairs into two independent four-value groups. - for (int group = 0; group < 8; group += 4) - { - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - buffer1[group + i], - buffer1[group + i + 2], - out buffer0[group + i], - out buffer0[group + i + 2]); - } - } - - // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. - for (int i = 0; i < 4; i++) - { - buffer1[i] = buffer0[i]; - } - - buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); - buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); - buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); - buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); - - // Stage 5 creates the four final butterfly pairs spanning the two groups. - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); - } - - // Stage 6 applies the remaining odd-angle rotations. - buffer1[0] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); - buffer1[1] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); - buffer1[2] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); - buffer1[3] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); - buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); - buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); - buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); - buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); - - ReadOnlySpan outputOrder = Adst8OutputOrder; - - // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. - for (int i = 0; i < 8; i++) - { - Store(ref values, outputStride, i, buffer1[outputOrder[i]]); - } - } - - /// - /// Applies the sixteen-point forward asymmetric discrete sine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Adst16( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. - buffer0[0] = Load(ref values, inputStride, 0); - buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 15)); - buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); - buffer0[3] = Load(ref values, inputStride, 8); - buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); - buffer0[5] = Load(ref values, inputStride, 12); - buffer0[6] = Load(ref values, inputStride, 4); - buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 11)); - buffer0[8] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); - buffer0[9] = Load(ref values, inputStride, 14); - buffer0[10] = Load(ref values, inputStride, 6); - buffer0[11] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 9)); - buffer0[12] = Load(ref values, inputStride, 2); - buffer0[13] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 13)); - buffer0[14] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); - buffer0[15] = Load(ref values, inputStride, 10); - - // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. - for (int group = 0; group < 16; group += 4) - { - buffer1[group] = buffer0[group]; - buffer1[group + 1] = buffer0[group + 1]; - Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); - } - - // Stage 3 combines adjacent pairs within each group of four. - for (int group = 0; group < 16; group += 4) - { - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - buffer1[group + i], - buffer1[group + i + 2], - out buffer0[group + i], - out buffer0[group + i + 2]); - } - } - - // Stage 4 rotates the upper pair of each eight-value group by pi/8. - for (int group = 0; group < 16; group += 8) - { - for (int i = 0; i < 4; i++) - { - buffer1[group + i] = buffer0[group + i]; - } - - buffer1[group + 4] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); - - buffer1[group + 5] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); - - buffer1[group + 6] = Av1ForwardTransformArithmetic.HalfButterfly( - -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); - - buffer1[group + 7] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); - } - - // Stage 5 combines the lower and upper quartets within each eight-value group. - for (int group = 0; group < 16; group += 8) - { - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - buffer1[group + i], - buffer1[group + i + 4], - out buffer0[group + i], - out buffer0[group + i + 4]); - } - } - - // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. - for (int i = 0; i < 8; i++) - { - buffer1[i] = buffer0[i]; - } - - buffer1[8] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); - buffer1[9] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); - buffer1[10] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); - buffer1[11] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); - buffer1[12] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); - buffer1[13] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); - buffer1[14] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); - buffer1[15] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); - - // Stage 7 creates the eight final butterfly pairs spanning both octets. - for (int i = 0; i < 8; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); - } - - ReadOnlySpan finalWeights = Adst16FinalWeights; - - // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order - // without allocating a per-call array or duplicating the complementary cosine-index calculation. - for (int pair = 0; pair < 8; pair++) - { - int first = finalWeights[pair]; - int second = 64 - first; - int index = pair * 2; - buffer1[index] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); - - buffer1[index + 1] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); - } - - ReadOnlySpan outputOrder = Adst16OutputOrder; - - // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. - for (int i = 0; i < 16; i++) - { - Store(ref values, outputStride, i, buffer1[outputOrder[i]]); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs deleted file mode 100644 index 44c091ec2..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Implements the sixteen-point forward DCT stage network. -/// -internal static partial class Av1ForwardTransformOperations -{ - /// - /// Applies the sixteen-point forward discrete cosine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Dct16( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. - for (int i = 0; i < 8; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, i), - Load(ref values, inputStride, 15 - i), - out buffer0[i], - out buffer0[15 - i]); - } - - // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); - } - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer0[10], - buffer0[13], - out buffer1[10], - out buffer1[13], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer0[11], - buffer0[12], - out buffer1[11], - out buffer1[12], - cosBit, - in rounding); - - // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); - } - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer1[5], - buffer1[6], - out buffer0[5], - out buffer0[6], - cosBit, - in rounding); - - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); - } - - // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. - Av1ForwardTransformArithmetic.Butterfly( - cospi[32], - cospi[32], - buffer0[0], - buffer0[1], - out TValue output0, - out TValue output8, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[16], - cospi[48], - buffer0[3], - buffer0[2], - out TValue output4, - out TValue output12, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); - Av1ForwardTransformArithmetic.Butterfly( - -cospi[16], - cospi[48], - buffer0[9], - buffer0[14], - out buffer1[9], - out buffer1[14], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[48], - -cospi[16], - buffer0[10], - buffer0[13], - out buffer1[10], - out buffer1[13], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[8], - cospi[56], - buffer1[7], - buffer1[4], - out TValue output2, - out TValue output14, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[40], - cospi[24], - buffer1[6], - buffer1[5], - out TValue output10, - out TValue output6, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); - - // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative - // coefficient permutation, so each rotation result is named by its final destination. - Av1ForwardTransformArithmetic.Butterfly( - cospi[4], - cospi[60], - buffer0[15], - buffer0[8], - out TValue output1, - out TValue output15, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[36], - cospi[28], - buffer0[14], - buffer0[9], - out TValue output9, - out TValue output7, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[20], - cospi[44], - buffer0[13], - buffer0[10], - out TValue output5, - out TValue output11, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[52], - cospi[12], - buffer0[12], - buffer0[11], - out TValue output13, - out TValue output3, - cosBit, - in rounding); - - Store(ref values, outputStride, 0, output0); - Store(ref values, outputStride, 1, output1); - Store(ref values, outputStride, 2, output2); - Store(ref values, outputStride, 3, output3); - Store(ref values, outputStride, 4, output4); - Store(ref values, outputStride, 5, output5); - Store(ref values, outputStride, 6, output6); - Store(ref values, outputStride, 7, output7); - Store(ref values, outputStride, 8, output8); - Store(ref values, outputStride, 9, output9); - Store(ref values, outputStride, 10, output10); - Store(ref values, outputStride, 11, output11); - Store(ref values, outputStride, 12, output12); - Store(ref values, outputStride, 13, output13); - Store(ref values, outputStride, 14, output14); - Store(ref values, outputStride, 15, output15); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs deleted file mode 100644 index 1a56d9372..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Implements the thirty-two-point forward DCT stage network. -/// -internal static partial class Av1ForwardTransformOperations -{ - /// - /// Applies the thirty-two-point forward discrete cosine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Dct32( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - // Stage 1 consumes the source block completely before any final coefficient is stored back into it. - for (int i = 0; i < 16; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, i), - Load(ref values, inputStride, 31 - i), - out buffer1[i], - out buffer1[31 - i]); - } - - // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. - for (int i = 0; i < 8; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); - } - - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer1[20 + i], - buffer1[27 - i], - out buffer0[20 + i], - out buffer0[27 - i], - cosBit, - in rounding); - } - - // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); - } - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer0[10], - buffer0[13], - out buffer1[10], - out buffer1[13], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer0[11], - buffer0[12], - out buffer1[11], - out buffer1[12], - cosBit, - in rounding); - - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); - } - - // Stage 4 continues the factorization as independent eight-value groups. - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); - } - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer1[5], - buffer1[6], - out buffer0[5], - out buffer0[6], - cosBit, - in rounding); - - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); - } - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[16], - cospi[48], - buffer1[18], - buffer1[29], - out buffer0[18], - out buffer0[29], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[16], - cospi[48], - buffer1[19], - buffer1[28], - out buffer0[19], - out buffer0[28], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[48], - -cospi[16], - buffer1[20], - buffer1[27], - out buffer0[20], - out buffer0[27], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[48], - -cospi[16], - buffer1[21], - buffer1[26], - out buffer0[21], - out buffer0[26], - cosBit, - in rounding); - - // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are - // retired directly to the block instead of being copied through a third workspace. - ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); - ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); - Av1ForwardTransformArithmetic.Butterfly( - -cospi[16], - cospi[48], - buffer0[9], - buffer0[14], - out buffer1[9], - out buffer1[14], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[48], - -cospi[16], - buffer0[10], - buffer0[13], - out buffer1[10], - out buffer1[13], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); - - // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. - ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); - ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); - Av1ForwardTransformArithmetic.Butterfly( - -cospi[8], - cospi[56], - buffer1[17], - buffer1[30], - out buffer0[17], - out buffer0[30], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[56], - -cospi[8], - buffer1[18], - buffer1[29], - out buffer0[18], - out buffer0[29], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[40], - cospi[24], - buffer1[21], - buffer1[26], - out buffer0[21], - out buffer0[26], - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - -cospi[24], - -cospi[40], - buffer1[22], - buffer1[25], - out buffer0[22], - out buffer0[25], - cosBit, - in rounding); - - // Stage 7 applies the pi/32 rotations to the next odd-frequency level. - ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); - ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); - ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); - ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); - - // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results - // are consumed by another arithmetic stage. - ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); - ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); - ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); - ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); - ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); - ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); - ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); - ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs deleted file mode 100644 index 88585396c..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Implements the sixty-four-point forward DCT stage network. -/// -internal static partial class Av1ForwardTransformOperations -{ - /// - /// Identifies coefficient positions whose final value resides in the first stage buffer. - /// - private const ulong Dct64Buffer0OutputMask = - (1UL << 2) | (1UL << 6) | (1UL << 8) | (1UL << 10) | (1UL << 14) | - (1UL << 18) | (1UL << 22) | (1UL << 24) | (1UL << 26) | (1UL << 30) | - (1UL << 34) | (1UL << 38) | (1UL << 40) | (1UL << 42) | (1UL << 46) | - (1UL << 50) | (1UL << 54) | (1UL << 56) | (1UL << 58) | (1UL << 62); - - /// - /// Gets the stage-nine rotation order for the middle quarter of the sixty-four-point DCT. - /// - private static ReadOnlySpan Dct64Stage9RotationOrder => [2, 34, 18, 50, 10, 42, 26, 58]; - - /// - /// Gets the stage-ten rotation order for the upper half of the sixty-four-point DCT. - /// - private static ReadOnlySpan Dct64Stage10RotationOrder => [1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61]; - - /// - /// Gets the mapping from coefficient order to the final staged value. - /// - private static ReadOnlySpan Dct64OutputOrder => - [ - 0, 32, 16, 48, 8, 40, 24, 56, 4, 36, 20, 52, 12, 44, 28, 60, - 2, 34, 18, 50, 10, 42, 26, 58, 6, 38, 22, 54, 14, 46, 30, 62, - 1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61, - 3, 35, 19, 51, 11, 43, 27, 59, 7, 39, 23, 55, 15, 47, 31, 63, - ]; - - /// - /// Applies the sixty-four-point forward discrete cosine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Dct64( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. - for (int i = 0; i < 32; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, i), - Load(ref values, inputStride, 63 - i), - out buffer0[i], - out buffer0[63 - i]); - } - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. - for (int i = 0; i < 16; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); - } - - for (int i = 0; i < 8; i++) - { - Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); - } - - // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. - for (int i = 0; i < 8; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); - } - - for (int i = 0; i < 4; i++) - { - Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); - } - - for (int i = 0; i < 8; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); - } - - for (int i = 0; i < 8; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); - } - - // Stage 4 continues the factorization as independent sixteen-value groups. - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); - } - - for (int i = 0; i < 2; i++) - { - Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); - } - - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); - Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); - } - - for (int i = 4; i < 8; i++) - { - Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); - } - - // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); - } - - Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); - - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); - Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); - } - - for (int i = 2; i < 4; i++) - { - Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); - } - - for (int i = 0; i < 4; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); - } - - // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. - Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); - Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); - Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); - Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); - - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); - Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); - Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); - } - - for (int i = 2; i < 4; i++) - { - Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); - Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); - } - - // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. - Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); - Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); - Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); - Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); - Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); - Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); - - for (int group = 0; group < 4; group++) - { - int offset = group * 8; - - for (int i = 0; i < 2; i++) - { - Av1ForwardTransformArithmetic.AddSubtract( - buffer0[32 + offset + i], - buffer1[35 + offset - i], - out buffer0[32 + offset + i], - out buffer0[35 + offset - i]); - - Av1ForwardTransformArithmetic.AddSubtract( - buffer0[39 + offset - i], - buffer1[36 + offset + i], - out buffer0[39 + offset - i], - out buffer0[36 + offset + i]); - } - } - - // Stage 8 applies the next level of odd-frequency rotations. - Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); - Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); - Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); - Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); - - Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); - - Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); - Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); - Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); - Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); - Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); - Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); - Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); - Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); - - // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the - // non-linear rotation order while keeping the constants in compile-time data. - for (int i = 0; i < 8; i++) - { - int low = 16 + i; - int high = 31 - i; - int odd = Dct64Stage9RotationOrder[i]; - Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); - } - - Av1ForwardTransformArithmetic.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); - - // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. - for (int i = 0; i < 16; i++) - { - int low = 32 + i; - int high = 63 - i; - int odd = Dct64Stage10RotationOrder[i]; - Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); - } - - // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and - // 16-31 remain in buffer0 at retirement, - // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. - ReadOnlySpan outputOrder = Dct64OutputOrder; - - for (int i = 0; i < 64; i++) - { - int sourceIndex = outputOrder[i]; - TValue value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; - - Store(ref values, outputStride, i, value); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs deleted file mode 100644 index 4b9ad0951..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Implements the eight-point forward DCT stage network. -/// -internal static partial class Av1ForwardTransformOperations -{ - /// - /// Applies the eight-point forward discrete cosine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Dct8( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer - // ownership, allowing the later even butterflies to write their final coefficients directly to the block. - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, 0), - Load(ref values, inputStride, 7), - out buffer0[0], - out buffer1[7]); - - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, 1), - Load(ref values, inputStride, 6), - out buffer0[1], - out buffer0[6]); - - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, 2), - Load(ref values, inputStride, 5), - out buffer0[2], - out buffer0[5]); - - Av1ForwardTransformArithmetic.AddSubtract( - Load(ref values, inputStride, 3), - Load(ref values, inputStride, 4), - out buffer0[3], - out buffer1[4]); - - Av1ForwardTransformArithmetic.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); - Av1ForwardTransformArithmetic.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], - cospi[32], - buffer0[5], - buffer0[6], - out buffer1[5], - out buffer1[6], - cosBit, - in rounding); - - // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. - Av1ForwardTransformArithmetic.Butterfly( - cospi[32], - cospi[32], - buffer1[0], - buffer1[1], - out TValue output0, - out TValue output4, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[16], - cospi[48], - buffer1[3], - buffer1[2], - out TValue output2, - out TValue output6, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); - Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); - - // Highway fuses the final two stages because no intermediate value is reused after either rotation. - Av1ForwardTransformArithmetic.Butterfly( - cospi[8], - cospi[56], - buffer0[7], - buffer0[4], - out TValue output1, - out TValue output7, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[40], - cospi[24], - buffer0[6], - buffer0[5], - out TValue output5, - out TValue output3, - cosBit, - in rounding); - - Store(ref values, outputStride, 0, output0); - Store(ref values, outputStride, 1, output1); - Store(ref values, outputStride, 2, output2); - Store(ref values, outputStride, 3, output3); - Store(ref values, outputStride, 4, output4); - Store(ref values, outputStride, 5, output5); - Store(ref values, outputStride, 6, output6); - Store(ref values, outputStride, 7, output7); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs deleted file mode 100644 index 67187adc2..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Implements the length-specific forward identity transform scaling. -/// -internal static partial class Av1ForwardTransformOperations -{ - /// - /// Applies the four-point forward identity transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Identity4( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 4, 1, 0); - - /// - /// Applies the eight-point forward identity transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Identity8( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 8, 0, 1); - - /// - /// Applies the sixteen-point forward identity transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Identity16( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 16, 2, 0); - - /// - /// Applies the thirty-two-point forward identity transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Identity32( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 32, 0, 2); - - /// - /// Applies the length-specific AV1 identity scaling directly to the strided transform block. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - /// The number of transform positions. - /// The square-root-of-two multiplier, or zero when power-of-two scaling applies. - /// The power-of-two scaling shift. - private static void Identity( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit, - int length, - int sqrt2Scale, - int leftShift) - where TValue : struct - { - _ = buffer0; - _ = buffer1; - _ = cosBit; - - // AV1 defines identity normalization by transform length: 4 and 16 use sqrt(2) scaling, while 8 and 32 - // are exact powers of two. Applying it in place matches Highway's row-oriented identity kernels. - for (int i = 0; i < length; i++) - { - TValue input = Load(ref values, inputStride, i); - TValue output = sqrt2Scale != 0 - ? Av1ForwardTransformArithmetic.MultiplyRound( - input, - sqrt2Scale * Av1Transform1dMath.NewSqrt2, - Av1Transform1dMath.NewSqrt2Bits) - : Av1ForwardTransformArithmetic.ShiftLeft(input, leftShift); - - Store(ref values, outputStride, i, output); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst16Operator.cs new file mode 100644 index 000000000..fa18febc8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst16Operator.cs @@ -0,0 +1,1077 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the sixteen-point forward ADST operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the sixteen-point forward asymmetric discrete sine transform. + /// + internal readonly struct Adst16Operator : IAv1ForwardTransform1dOperator + { + /// + /// Gets the first cosine index for each final rotation. + /// + private static ReadOnlySpan FinalWeights => [2, 10, 18, 26, 34, 42, 50, 58]; + + /// + /// Gets the fixed coefficient permutation. + /// + private static ReadOnlySpan OutputOrder => [1, 14, 3, 12, 5, 10, 7, 8, 9, 6, 11, 4, 13, 2, 15, 0]; + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); + buffer0[3] = Load(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); + buffer0[5] = Load(ref values, inputStride, 12); + buffer0[6] = Load(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); + buffer0[9] = Load(ref values, inputStride, 14); + buffer0[10] = Load(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 9)); + buffer0[12] = Load(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); + buffer0[15] = Load(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); + buffer0[3] = Load(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); + buffer0[5] = Load(ref values, inputStride, 12); + buffer0[6] = Load(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); + buffer0[9] = Load(ref values, inputStride, 14); + buffer0[10] = Load(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 9)); + buffer0[12] = Load(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); + buffer0[15] = Load(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[3] = Load>(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[5] = Load>(ref values, inputStride, 12); + buffer0[6] = Load>(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[9] = Load>(ref values, inputStride, 14); + buffer0[10] = Load>(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 9)); + buffer0[12] = Load>(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + buffer0[15] = Load>(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic>.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[3] = Load>(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[5] = Load>(ref values, inputStride, 12); + buffer0[6] = Load>(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[9] = Load>(ref values, inputStride, 14); + buffer0[10] = Load>(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 9)); + buffer0[12] = Load>(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + buffer0[15] = Load>(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic>.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[3] = Load>(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[5] = Load>(ref values, inputStride, 12); + buffer0[6] = Load>(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[9] = Load>(ref values, inputStride, 14); + buffer0[10] = Load>(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 9)); + buffer0[12] = Load>(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + buffer0[15] = Load>(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic>.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[3] = Load>(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[5] = Load>(ref values, inputStride, 12); + buffer0[6] = Load>(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[9] = Load>(ref values, inputStride, 14); + buffer0[10] = Load>(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 9)); + buffer0[12] = Load>(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + buffer0[15] = Load>(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic>.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[3] = Load>(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[5] = Load>(ref values, inputStride, 12); + buffer0[6] = Load>(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[9] = Load>(ref values, inputStride, 14); + buffer0[10] = Load>(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 9)); + buffer0[12] = Load>(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + buffer0[15] = Load>(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic>.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[3] = Load>(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[5] = Load>(ref values, inputStride, 12); + buffer0[6] = Load>(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[9] = Load>(ref values, inputStride, 14); + buffer0[10] = Load>(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 9)); + buffer0[12] = Load>(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + buffer0[15] = Load>(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + buffer1[group + i] = buffer0[group + i]; + } + + buffer1[group + 4] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic>.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[8] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); + } + + ReadOnlySpan finalWeights = FinalWeights; + + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. + for (int pair = 0; pair < 8; pair++) + { + int first = finalWeights[pair]; + int second = 64 - first; + int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); + + buffer1[index + 1] = Av1ForwardTransformArithmetic>.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); + } + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 16; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst4Operator.cs new file mode 100644 index 000000000..6416b8a6c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst4Operator.cs @@ -0,0 +1,411 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the four-point forward ADST operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the four-point forward asymmetric discrete sine transform. + /// + internal readonly struct Adst4Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + int input0 = Load(ref values, inputStride, 0); + int input1 = Load(ref values, inputStride, 1); + int input2 = Load(ref values, inputStride, 2); + int input3 = Load(ref values, inputStride, 3); + int input01 = Av1ForwardTransformArithmetic.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + int output0 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + int output1 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + int output2 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + int output3 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + short input0 = Load(ref values, inputStride, 0); + short input1 = Load(ref values, inputStride, 1); + short input2 = Load(ref values, inputStride, 2); + short input3 = Load(ref values, inputStride, 3); + short input01 = Av1ForwardTransformArithmetic.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + short output0 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + short output1 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + short output2 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + short output3 = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + Vector128 input0 = Load>(ref values, inputStride, 0); + Vector128 input1 = Load>(ref values, inputStride, 1); + Vector128 input2 = Load>(ref values, inputStride, 2); + Vector128 input3 = Load>(ref values, inputStride, 3); + Vector128 input01 = Av1ForwardTransformArithmetic>.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + Vector128 output0 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + Vector128 output1 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + Vector128 output2 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + Vector128 output3 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + Vector256 input0 = Load>(ref values, inputStride, 0); + Vector256 input1 = Load>(ref values, inputStride, 1); + Vector256 input2 = Load>(ref values, inputStride, 2); + Vector256 input3 = Load>(ref values, inputStride, 3); + Vector256 input01 = Av1ForwardTransformArithmetic>.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + Vector256 output0 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + Vector256 output1 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + Vector256 output2 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + Vector256 output3 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + Vector512 input0 = Load>(ref values, inputStride, 0); + Vector512 input1 = Load>(ref values, inputStride, 1); + Vector512 input2 = Load>(ref values, inputStride, 2); + Vector512 input3 = Load>(ref values, inputStride, 3); + Vector512 input01 = Av1ForwardTransformArithmetic>.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + Vector512 output0 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + Vector512 output1 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + Vector512 output2 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + Vector512 output3 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + Vector128 input0 = Load>(ref values, inputStride, 0); + Vector128 input1 = Load>(ref values, inputStride, 1); + Vector128 input2 = Load>(ref values, inputStride, 2); + Vector128 input3 = Load>(ref values, inputStride, 3); + Vector128 input01 = Av1ForwardTransformArithmetic>.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + Vector128 output0 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + Vector128 output1 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + Vector128 output2 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + Vector128 output3 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + Vector256 input0 = Load>(ref values, inputStride, 0); + Vector256 input1 = Load>(ref values, inputStride, 1); + Vector256 input2 = Load>(ref values, inputStride, 2); + Vector256 input3 = Load>(ref values, inputStride, 3); + Vector256 input01 = Av1ForwardTransformArithmetic>.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + Vector256 output0 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + Vector256 output1 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + Vector256 output2 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + Vector256 output3 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + Vector512 input0 = Load>(ref values, inputStride, 0); + Vector512 input1 = Load>(ref values, inputStride, 1); + Vector512 input2 = Load>(ref values, inputStride, 2); + Vector512 input3 = Load>(ref values, inputStride, 3); + Vector512 input01 = Av1ForwardTransformArithmetic>.Add(input0, input1); + + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + Vector512 output0 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + + Vector512 output1 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + + Vector512 output2 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + Vector512 output3 = Av1ForwardTransformArithmetic>.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst8Operator.cs new file mode 100644 index 000000000..17bdaa53d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Adst8Operator.cs @@ -0,0 +1,656 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the eight-point forward ADST operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the eight-point forward asymmetric discrete sine transform. + /// + internal readonly struct Adst8Operator : IAv1ForwardTransform1dOperator + { + /// + /// Gets the fixed coefficient permutation. + /// + private static ReadOnlySpan OutputOrder => [1, 6, 3, 4, 5, 2, 7, 0]; + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); + buffer0[3] = Load(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); + buffer0[5] = Load(ref values, inputStride, 6); + buffer0[6] = Load(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); + buffer0[3] = Load(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); + buffer0[5] = Load(ref values, inputStride, 6); + buffer0[6] = Load(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[3] = Load>(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[5] = Load>(ref values, inputStride, 6); + buffer0[6] = Load>(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[3] = Load>(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[5] = Load>(ref values, inputStride, 6); + buffer0[6] = Load>(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[3] = Load>(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[5] = Load>(ref values, inputStride, 6); + buffer0[6] = Load>(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[3] = Load>(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[5] = Load>(ref values, inputStride, 6); + buffer0[6] = Load>(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[3] = Load>(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[5] = Load>(ref values, inputStride, 6); + buffer0[6] = Load>(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load>(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 3)); + buffer0[3] = Load>(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 1)); + buffer0[5] = Load>(ref values, inputStride, 6); + buffer0[6] = Load>(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic>.Negate(Load>(ref values, inputStride, 5)); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. + for (int i = 0; i < 4; i++) + { + buffer1[i] = buffer0[i]; + } + + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); + } + + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic>.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct16Operator.cs new file mode 100644 index 000000000..ce7497096 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct16Operator.cs @@ -0,0 +1,1603 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Implements the sixteen-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the sixteen-point forward transform for every supported lane width. + /// + internal readonly struct Dct16Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out int output0, + out int output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out int output4, + out int output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out int output2, + out int output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out int output10, + out int output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out int output1, + out int output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out int output9, + out int output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out int output5, + out int output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out int output13, + out int output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out short output0, + out short output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out short output4, + out short output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out short output2, + out short output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out short output10, + out short output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out short output1, + out short output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out short output9, + out short output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out short output5, + out short output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out short output13, + out short output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector128 output0, + out Vector128 output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector128 output4, + out Vector128 output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out Vector128 output2, + out Vector128 output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out Vector128 output10, + out Vector128 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out Vector128 output1, + out Vector128 output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out Vector128 output9, + out Vector128 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out Vector128 output5, + out Vector128 output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out Vector128 output13, + out Vector128 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector256 output0, + out Vector256 output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector256 output4, + out Vector256 output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out Vector256 output2, + out Vector256 output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out Vector256 output10, + out Vector256 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out Vector256 output1, + out Vector256 output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out Vector256 output9, + out Vector256 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out Vector256 output5, + out Vector256 output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out Vector256 output13, + out Vector256 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector512 output0, + out Vector512 output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector512 output4, + out Vector512 output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out Vector512 output2, + out Vector512 output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out Vector512 output10, + out Vector512 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out Vector512 output1, + out Vector512 output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out Vector512 output9, + out Vector512 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out Vector512 output5, + out Vector512 output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out Vector512 output13, + out Vector512 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector128 output0, + out Vector128 output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector128 output4, + out Vector128 output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out Vector128 output2, + out Vector128 output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out Vector128 output10, + out Vector128 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out Vector128 output1, + out Vector128 output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out Vector128 output9, + out Vector128 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out Vector128 output5, + out Vector128 output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out Vector128 output13, + out Vector128 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector256 output0, + out Vector256 output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector256 output4, + out Vector256 output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out Vector256 output2, + out Vector256 output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out Vector256 output10, + out Vector256 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out Vector256 output1, + out Vector256 output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out Vector256 output9, + out Vector256 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out Vector256 output5, + out Vector256 output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out Vector256 output13, + out Vector256 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector512 output0, + out Vector512 output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector512 output4, + out Vector512 output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out Vector512 output2, + out Vector512 output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out Vector512 output10, + out Vector512 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out Vector512 output1, + out Vector512 output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out Vector512 output9, + out Vector512 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out Vector512 output5, + out Vector512 output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out Vector512 output13, + out Vector512 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct32Operator.cs new file mode 100644 index 000000000..1624d7bbb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct32Operator.cs @@ -0,0 +1,1963 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Implements the thirty-two-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the thirty-two-point forward transform for every supported lane width. + /// + internal readonly struct Dct32Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); + } + + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + } + + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct4Operator.cs new file mode 100644 index 000000000..2987fc3db --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct4Operator.cs @@ -0,0 +1,395 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the four-point forward DCT operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the four-point forward discrete cosine transform. + /// + internal readonly struct Dct4Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + int input0 = Load(ref values, inputStride, 0); + int input1 = Load(ref values, inputStride, 1); + int input2 = Load(ref values, inputStride, 2); + int input3 = Load(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out int output0, + out int output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out int output1, + out int output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + short input0 = Load(ref values, inputStride, 0); + short input1 = Load(ref values, inputStride, 1); + short input2 = Load(ref values, inputStride, 2); + short input3 = Load(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out short output0, + out short output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out short output1, + out short output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + Vector128 input0 = Load>(ref values, inputStride, 0); + Vector128 input1 = Load>(ref values, inputStride, 1); + Vector128 input2 = Load>(ref values, inputStride, 2); + Vector128 input3 = Load>(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector128 output0, + out Vector128 output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector128 output1, + out Vector128 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + Vector256 input0 = Load>(ref values, inputStride, 0); + Vector256 input1 = Load>(ref values, inputStride, 1); + Vector256 input2 = Load>(ref values, inputStride, 2); + Vector256 input3 = Load>(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector256 output0, + out Vector256 output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector256 output1, + out Vector256 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + Vector512 input0 = Load>(ref values, inputStride, 0); + Vector512 input1 = Load>(ref values, inputStride, 1); + Vector512 input2 = Load>(ref values, inputStride, 2); + Vector512 input3 = Load>(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector512 output0, + out Vector512 output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector512 output1, + out Vector512 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + Vector128 input0 = Load>(ref values, inputStride, 0); + Vector128 input1 = Load>(ref values, inputStride, 1); + Vector128 input2 = Load>(ref values, inputStride, 2); + Vector128 input3 = Load>(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector128 output0, + out Vector128 output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector128 output1, + out Vector128 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + Vector256 input0 = Load>(ref values, inputStride, 0); + Vector256 input1 = Load>(ref values, inputStride, 1); + Vector256 input2 = Load>(ref values, inputStride, 2); + Vector256 input3 = Load>(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector256 output0, + out Vector256 output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector256 output1, + out Vector256 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + Vector512 input0 = Load>(ref values, inputStride, 0); + Vector512 input1 = Load>(ref values, inputStride, 1); + Vector512 input2 = Load>(ref values, inputStride, 2); + Vector512 input3 = Load>(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic>.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic>.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out Vector512 output0, + out Vector512 output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out Vector512 output1, + out Vector512 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct64Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct64Operator.cs new file mode 100644 index 000000000..b4f52dfed --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct64Operator.cs @@ -0,0 +1,1929 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Implements the sixty-four-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the sixty-four-point forward transform for every supported lane width. + /// + internal readonly struct Dct64Operator : IAv1ForwardTransform1dOperator + { + /// + /// Identifies coefficient positions whose final value resides in the first stage buffer. + /// + private const ulong Dct64Buffer0OutputMask = + (1UL << 2) | (1UL << 6) | (1UL << 8) | (1UL << 10) | (1UL << 14) | + (1UL << 18) | (1UL << 22) | (1UL << 24) | (1UL << 26) | (1UL << 30) | + (1UL << 34) | (1UL << 38) | (1UL << 40) | (1UL << 42) | (1UL << 46) | + (1UL << 50) | (1UL << 54) | (1UL << 56) | (1UL << 58) | (1UL << 62); + + /// + /// Gets the stage-nine rotation order for the middle quarter of the transform. + /// + private static ReadOnlySpan Dct64Stage9RotationOrder => [2, 34, 18, 50, 10, 42, 26, 58]; + + /// + /// Gets the stage-ten rotation order for the upper half of the transform. + /// + private static ReadOnlySpan Dct64Stage10RotationOrder => [1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61]; + + /// + /// Gets the mapping from coefficient order to the final staged value. + /// + private static ReadOnlySpan Dct64OutputOrder => + [ + 0, 32, 16, 48, 8, 40, 24, 56, 4, 36, 20, 52, 12, 44, 28, 60, + 2, 34, 18, 50, 10, 42, 26, 58, 6, 38, 22, 54, 14, 46, 30, 62, + 1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61, + 3, 35, 19, 51, 11, 43, 27, 59, 7, 39, 23, 55, 15, 47, 31, 63, + ]; + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + int value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + short value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + Vector128 value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + Vector256 value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + Vector512 value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + Vector128 value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + Vector256 value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, i), + Load>(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); + } + + for (int i = 0; i < 8; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); + } + + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); + } + + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + for (int i = 0; i < 2; i++) + { + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + for (int i = 4; i < 8; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); + } + + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic>.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + Vector512 value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct8Operator.cs new file mode 100644 index 000000000..c2223d45f --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Dct8Operator.cs @@ -0,0 +1,859 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Implements the eight-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the eight-point forward transform for every supported lane width. + /// + internal readonly struct Dct8Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 0), + Load(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 1), + Load(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 2), + Load(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 3), + Load(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out int output0, + out int output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out int output2, + out int output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out int output1, + out int output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out int output5, + out int output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 0), + Load(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 1), + Load(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 2), + Load(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 3), + Load(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out short output0, + out short output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out short output2, + out short output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out short output1, + out short output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out short output5, + out short output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 0), + Load>(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 1), + Load>(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 2), + Load>(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 3), + Load>(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out Vector128 output0, + out Vector128 output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out Vector128 output2, + out Vector128 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out Vector128 output1, + out Vector128 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out Vector128 output5, + out Vector128 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 0), + Load>(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 1), + Load>(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 2), + Load>(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 3), + Load>(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out Vector256 output0, + out Vector256 output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out Vector256 output2, + out Vector256 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out Vector256 output1, + out Vector256 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out Vector256 output5, + out Vector256 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 0), + Load>(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 1), + Load>(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 2), + Load>(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 3), + Load>(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out Vector512 output0, + out Vector512 output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out Vector512 output2, + out Vector512 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out Vector512 output1, + out Vector512 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out Vector512 output5, + out Vector512 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 0), + Load>(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 1), + Load>(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 2), + Load>(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 3), + Load>(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out Vector128 output0, + out Vector128 output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out Vector128 output2, + out Vector128 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out Vector128 output1, + out Vector128 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out Vector128 output5, + out Vector128 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 0), + Load>(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 1), + Load>(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 2), + Load>(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 3), + Load>(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out Vector256 output0, + out Vector256 output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out Vector256 output2, + out Vector256 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out Vector256 output1, + out Vector256 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out Vector256 output5, + out Vector256 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic>.CreateRounding(cosBit); + + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 0), + Load>(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 1), + Load>(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 2), + Load>(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic>.AddSubtract( + Load>(ref values, inputStride, 3), + Load>(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic>.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out Vector512 output0, + out Vector512 output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out Vector512 output2, + out Vector512 output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic>.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); + + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic>.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out Vector512 output1, + out Vector512 output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic>.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out Vector512 output5, + out Vector512 output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity16Operator.cs new file mode 100644 index 000000000..51b0878f1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity16Operator.cs @@ -0,0 +1,235 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the sixteen-point forward identity operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the sixteen-point forward identity transform. + /// + internal readonly struct Identity16Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + int input = Load(ref values, inputStride, i); + int output = Av1ForwardTransformArithmetic.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + short input = Load(ref values, inputStride, i); + short output = Av1ForwardTransformArithmetic.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 16; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 2 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity32Operator.cs new file mode 100644 index 000000000..3d6a179f5 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity32Operator.cs @@ -0,0 +1,211 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the thirty-two-point forward identity operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the thirty-two-point forward identity transform. + /// + internal readonly struct Identity32Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + int input = Load(ref values, inputStride, i); + int output = Av1ForwardTransformArithmetic.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + short input = Load(ref values, inputStride, i); + short output = Av1ForwardTransformArithmetic.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 32; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 2); + + Store(ref values, outputStride, i, output); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity4Operator.cs new file mode 100644 index 000000000..cb2e20676 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity4Operator.cs @@ -0,0 +1,235 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the four-point forward identity operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the four-point forward identity transform. + /// + internal readonly struct Identity4Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + int input = Load(ref values, inputStride, i); + int output = Av1ForwardTransformArithmetic.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + short input = Load(ref values, inputStride, i); + short output = Av1ForwardTransformArithmetic.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 4; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.MultiplyRound( + input, + 1 * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + Store(ref values, outputStride, i, output); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity8Operator.cs new file mode 100644 index 000000000..ec8668710 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Identity8Operator.cs @@ -0,0 +1,211 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the eight-point forward identity operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Implements the eight-point forward identity transform. + /// + internal readonly struct Identity8Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + int input = Load(ref values, inputStride, i); + int output = Av1ForwardTransformArithmetic.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + short input = Load(ref values, inputStride, i); + short output = Av1ForwardTransformArithmetic.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + Vector128 input = Load>(ref values, inputStride, i); + Vector128 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + Vector256 input = Load>(ref values, inputStride, i); + Vector256 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit) + { + _ = buffer0; + _ = buffer1; + _ = cosBit; + + // The length-specific normalization is applied directly in the semantic operator so each scalar + // or SIMD overload retains the exact AV1 identity-transform arithmetic without a forwarding layer. + for (int i = 0; i < 8; i++) + { + Vector512 input = Load>(ref values, inputStride, i); + Vector512 output = Av1ForwardTransformArithmetic>.ShiftLeft(input, 1); + + Store(ref values, outputStride, i, output); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operations.cs similarity index 64% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operations.cs index bd53e36ce..fa19fd531 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operations.cs @@ -2,71 +2,15 @@ // Licensed under the Six Labors Split License. using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// -/// Implements the four-point forward DCT stage network. +/// Defines shared forward-transform storage and rotation primitives. /// -internal static partial class Av1ForwardTransformOperations +internal static partial class Av1ForwardTransformer { - /// - /// Applies the four-point forward discrete cosine transform to every independent lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static void Dct4( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - - TValue input0 = Load(ref values, inputStride, 0); - TValue input1 = Load(ref values, inputStride, 1); - TValue input2 = Load(ref values, inputStride, 2); - TValue input3 = Load(ref values, inputStride, 3); - - // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain - // Highway's saturating add/subtract behavior before the widening butterfly multiplication. - Av1ForwardTransformArithmetic.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); - Av1ForwardTransformArithmetic.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); - Av1ForwardTransformArithmetic.Butterfly( - cospi[32], - cospi[32], - buffer0[0], - buffer0[1], - out TValue output0, - out TValue output2, - cosBit, - in rounding); - - Av1ForwardTransformArithmetic.Butterfly( - cospi[16], - cospi[48], - buffer0[3], - buffer0[2], - out TValue output1, - out TValue output3, - cosBit, - in rounding); - - Store(ref values, outputStride, 0, output0); - Store(ref values, outputStride, 1, output1); - Store(ref values, outputStride, 2, output2); - Store(ref values, outputStride, 3, output3); - } - /// /// Loads one scalar or SIMD transform value from strided block storage. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operator.cs new file mode 100644 index 000000000..fc4cace53 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformer.Operator.cs @@ -0,0 +1,158 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the operator contract for one-dimensional AV1 forward transforms. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the scalar and SIMD arithmetic for one AV1 forward transform. + /// + /// + /// Every overload applies the same stage network to independent transform axes. The family traversal selects one + /// concrete lane width, while the closed semantic operator lets the JIT resolve the static call before the stages. + /// + internal interface IAv1ForwardTransform1dOperator + { + /// + /// Transforms one expanded axis without hardware vectorization. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit); + + /// + /// Transforms one packed axis without hardware vectorization. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit); + + /// + /// Transforms eight packed axes in parallel. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit); + + /// + /// Transforms sixteen packed axes in parallel. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit); + + /// + /// Transforms thirty-two packed axes in parallel. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit); + + /// + /// Transforms four expanded axes in parallel. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit); + + /// + /// Transforms eight expanded axes in parallel. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit); + + /// + /// Transforms sixteen expanded axes in parallel. + /// + /// The first value in the strided transform storage. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first transform-stage workspace buffer. + /// The second transform-stage workspace buffer. + /// The fixed-point precision of the transform constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector> buffer0, + ref Av1TransformVector> buffer1, + int cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs deleted file mode 100644 index 5216a8900..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixteen-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity16Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs deleted file mode 100644 index f15c0b868..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the thirty-two-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity32Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs deleted file mode 100644 index 89236df3d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the four-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity4Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs deleted file mode 100644 index 96a295221..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the eight-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity8Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs deleted file mode 100644 index 6aa68233d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type. -/// -/// -/// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type -/// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch -/// inside the transform stages. -/// -internal interface IAv1ForwardTransform1dOperator -{ - /// - /// Transforms the independent axes stored in each value lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static abstract void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct; -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs deleted file mode 100644 index 7e2244cc6..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Defines how inverse-transform residuals are added to a decoded sample representation. -/// -/// -/// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions, -/// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable -/// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize. -/// -/// The decoded sample storage type. -internal interface IAv1InverseTransformOutputOperator - where TSample : unmanaged -{ - /// - /// Adds one residual to a predicted sample and clips the result to the coded bit depth. - /// - /// The predicted sample. - /// The inverse-transform residual. - /// The coded sample bit depth. - /// The reconstructed sample. - public static abstract TSample Add(TSample prediction, int residual, int bitDepth); - - /// - /// Adds four residuals to four predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The four inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth); - - /// - /// Adds eight residuals to eight predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The eight inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs deleted file mode 100644 index 09ecb8a3b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional transform. -/// -/// -/// Each overload performs the same staged fixed-point transform. In the SIMD overloads, each vector field identifies -/// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore -/// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage -/// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the -/// JIT to specialize every static interface call outside the stage network. -/// -internal interface IAv1Transform1dOperator -{ - /// - /// Transforms one axis when hardware vectorization is unavailable. - /// - /// The source values for the transform axis. - /// The destination values for the transform axis. - /// The fixed stage storage for the transform axis. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange); - - /// - /// Transforms four independent axes in parallel. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed stage storage for four transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); - - /// - /// Transforms eight independent axes in parallel. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed stage storage for eight transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs deleted file mode 100644 index e7e1a2a9f..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Adst16Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen spatial-domain residual values. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[15]; - output[1] = input[0]; - output[2] = input[13]; - output[3] = input[2]; - output[4] = input[11]; - output[5] = input[4]; - output[6] = input[9]; - output[7] = input[6]; - output[8] = input[7]; - output[9] = input[8]; - output[10] = input[5]; - output[11] = input[10]; - output[12] = input[3]; - output[13] = input[12]; - output[14] = input[1]; - output[15] = input[14]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[8]; - output[2] = step[12]; - output[3] = -step[4]; - output[4] = step[6]; - output[5] = -step[14]; - output[6] = step[10]; - output[7] = -step[2]; - output[8] = step[3]; - output[9] = -step[11]; - output[10] = step[15]; - output[11] = -step[7]; - output[12] = step[5]; - output[13] = -step[13]; - output[14] = step[9]; - output[15] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V15; - output.V1 = input.V0; - output.V2 = input.V13; - output.V3 = input.V2; - output.V4 = input.V11; - output.V5 = input.V4; - output.V6 = input.V9; - output.V7 = input.V6; - output.V8 = input.V7; - output.V9 = input.V8; - output.V10 = input.V5; - output.V11 = input.V10; - output.V12 = input.V3; - output.V13 = input.V12; - output.V14 = input.V1; - output.V15 = input.V14; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V8; - output.V2 = step.V12; - output.V3 = -step.V4; - output.V4 = step.V6; - output.V5 = -step.V14; - output.V6 = step.V10; - output.V7 = -step.V2; - output.V8 = step.V3; - output.V9 = -step.V11; - output.V10 = step.V15; - output.V11 = -step.V7; - output.V12 = step.V5; - output.V13 = -step.V13; - output.V14 = step.V9; - output.V15 = -step.V1; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V15; - output.V1 = input.V0; - output.V2 = input.V13; - output.V3 = input.V2; - output.V4 = input.V11; - output.V5 = input.V4; - output.V6 = input.V9; - output.V7 = input.V6; - output.V8 = input.V7; - output.V9 = input.V8; - output.V10 = input.V5; - output.V11 = input.V10; - output.V12 = input.V3; - output.V13 = input.V12; - output.V14 = input.V1; - output.V15 = input.V14; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V8; - output.V2 = step.V12; - output.V3 = -step.V4; - output.V4 = step.V6; - output.V5 = -step.V14; - output.V6 = step.V10; - output.V7 = -step.V2; - output.V8 = step.V3; - output.V9 = -step.V11; - output.V10 = step.V15; - output.V11 = -step.V7; - output.V12 = step.V5; - output.V13 = -step.V13; - output.V14 = step.V9; - output.V15 = -step.V1; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs deleted file mode 100644 index 558064696..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Adst4Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform. - /// - /// The four frequency-domain coefficients. - /// The four spatial-domain residual values. - /// The stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the sine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - - // libaom widens the complete four-point factorization because the products retain their fixed-point scale - // until the final shift. The stage buffer is therefore unnecessary for this transform size. - long x0 = input[0]; - long x1 = input[1]; - long x2 = input[2]; - long x3 = input[3]; - - _ = step; - _ = stageRange; - - // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel. - if ((x0 | x1 | x2 | x3) == 0) - { - output[..4].Clear(); - return; - } - - // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3. - long s0 = sinpi[1] * x0; - long s1 = sinpi[2] * x0; - long s2 = sinpi[3] * x1; - long s3 = sinpi[4] * x2; - long s4 = sinpi[1] * x2; - long s5 = sinpi[2] * x3; - long s6 = sinpi[4] * x3; - long s7 = (x0 - x2) + x3; - - // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding. - s0 += s3; - s1 -= s4; - s3 = s2; - s2 = sinpi[3] * s7; - s0 += s5; - s1 -= s6; - x0 = s0 + s3; - x1 = s1 + s3; - x2 = s2; - x3 = (s0 + s1) - s3; - - output[0] = Av1Math.RoundShift(x0, cosBit); - output[1] = Av1Math.RoundShift(x1, cosBit); - output[2] = Av1Math.RoundShift(x2, cosBit); - output[3] = Av1Math.RoundShift(x3, cosBit); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit, stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount); - _ = step; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit, stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount); - _ = step; - } - - /// - /// Applies the inverse four-point matrix to four independent axes. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed-point precision of the sine constants. - /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. - private static void TransformCore( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - int cosBit, - bool widenedRound) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector128 x0 = input.V0; - Vector128 x1 = input.V1; - Vector128 x2 = input.V2; - Vector128 x3 = input.V3; - - // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row - // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow - // a valid Int32 fixed-point sum. - if (widenedRound) - { - output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - return; - } - - output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the inverse four-point matrix to eight independent axes. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed-point precision of the sine constants. - /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. - private static void TransformCore( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - int cosBit, - bool widenedRound) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector256 x0 = input.V0; - Vector256 x1 = input.V1; - Vector256 x2 = input.V2; - Vector256 x3 = input.V3; - - if (widenedRound) - { - output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - return; - } - - output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs deleted file mode 100644 index 0f998e26d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Adst8Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform. - /// - /// The eight frequency-domain coefficients. - /// The eight spatial-domain residual values. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[7]; - output[1] = input[0]; - output[2] = input[5]; - output[3] = input[2]; - output[4] = input[3]; - output[5] = input[4]; - output[6] = input[1]; - output[7] = input[6]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[4]; - output[2] = step[6]; - output[3] = -step[2]; - output[4] = step[3]; - output[5] = -step[7]; - output[6] = step[5]; - output[7] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V7; - output.V1 = input.V0; - output.V2 = input.V5; - output.V3 = input.V2; - output.V4 = input.V3; - output.V5 = input.V4; - output.V6 = input.V1; - output.V7 = input.V6; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V4; - output.V2 = step.V6; - output.V3 = -step.V2; - output.V4 = step.V3; - output.V5 = -step.V7; - output.V6 = step.V5; - output.V7 = -step.V1; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V7; - output.V1 = input.V0; - output.V2 = input.V5; - output.V3 = input.V2; - output.V4 = input.V3; - output.V5 = input.V4; - output.V6 = input.V1; - output.V7 = input.V6; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V4; - output.V2 = step.V6; - output.V3 = -step.V2; - output.V4 = step.V3; - output.V5 = -step.V7; - output.V6 = step.V5; - output.V7 = -step.V1; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs deleted file mode 100644 index 442177986..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs +++ /dev/null @@ -1,475 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 16-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Dct16Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 16-point AV1 inverse discrete cosine transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen spatial-domain residual values. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[8]; - output[2] = input[4]; - output[3] = input[12]; - output[4] = input[2]; - output[5] = input[10]; - output[6] = input[6]; - output[7] = input[14]; - output[8] = input[1]; - output[9] = input[9]; - output[10] = input[5]; - output[11] = input[13]; - output[12] = input[3]; - output[13] = input[11]; - output[14] = input[7]; - output[15] = input[15]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V8; - output.V2 = input.V4; - output.V3 = input.V12; - output.V4 = input.V2; - output.V5 = input.V10; - output.V6 = input.V6; - output.V7 = input.V14; - output.V8 = input.V1; - output.V9 = input.V9; - output.V10 = input.V5; - output.V11 = input.V13; - output.V12 = input.V3; - output.V13 = input.V11; - output.V14 = input.V7; - output.V15 = input.V15; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V8; - output.V2 = input.V4; - output.V3 = input.V12; - output.V4 = input.V2; - output.V5 = input.V10; - output.V6 = input.V6; - output.V7 = input.V14; - output.V8 = input.V1; - output.V9 = input.V9; - output.V10 = input.V5; - output.V11 = input.V13; - output.V12 = input.V3; - output.V13 = input.V11; - output.V14 = input.V7; - output.V15 = input.V15; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs deleted file mode 100644 index d768f9620..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs +++ /dev/null @@ -1,1027 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 32-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Dct32Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 32-point AV1 inverse discrete cosine transform. - /// - /// The 32 frequency-domain coefficients. - /// The 32 spatial-domain residual values. - /// The 32-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[16]; - output[2] = input[8]; - output[3] = input[24]; - output[4] = input[4]; - output[5] = input[20]; - output[6] = input[12]; - output[7] = input[28]; - output[8] = input[2]; - output[9] = input[18]; - output[10] = input[10]; - output[11] = input[26]; - output[12] = input[6]; - output[13] = input[22]; - output[14] = input[14]; - output[15] = input[30]; - output[16] = input[1]; - output[17] = input[17]; - output[18] = input[9]; - output[19] = input[25]; - output[20] = input[5]; - output[21] = input[21]; - output[22] = input[13]; - output[23] = input[29]; - output[24] = input[3]; - output[25] = input[19]; - output[26] = input[11]; - output[27] = input[27]; - output[28] = input[7]; - output[29] = input[23]; - output[30] = input[15]; - output[31] = input[31]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); - output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); - output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); - output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); - output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); - output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); - output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); - output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); - output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); - step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); - step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); - step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); - step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); - step[31] = output[31]; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); - output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); - output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); - output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); - output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); - output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); - output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); - step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); - step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); - step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); - output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); - output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); - output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); - output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); - step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); - step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); - step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); - output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); - output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); - output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V16; - output.V2 = input.V8; - output.V3 = input.V24; - output.V4 = input.V4; - output.V5 = input.V20; - output.V6 = input.V12; - output.V7 = input.V28; - output.V8 = input.V2; - output.V9 = input.V18; - output.V10 = input.V10; - output.V11 = input.V26; - output.V12 = input.V6; - output.V13 = input.V22; - output.V14 = input.V14; - output.V15 = input.V30; - output.V16 = input.V1; - output.V17 = input.V17; - output.V18 = input.V9; - output.V19 = input.V25; - output.V20 = input.V5; - output.V21 = input.V21; - output.V22 = input.V13; - output.V23 = input.V29; - output.V24 = input.V3; - output.V25 = input.V19; - output.V26 = input.V11; - output.V27 = input.V27; - output.V28 = input.V7; - output.V29 = input.V23; - output.V30 = input.V15; - output.V31 = input.V31; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); - step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); - step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); - output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); - output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); - output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); - output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); - output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); - output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); - output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); - output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); - output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); - output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); - step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); - step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); - step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); - step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); - step.V16 = output.V16; - step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); - step.V31 = output.V31; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - output.V8 = step.V8; - output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); - output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); - output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); - output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); - output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); - output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); - output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); - step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); - step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); - step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); - output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); - output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); - output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); - output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); - step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); - step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); - step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); - output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); - output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); - output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V16; - output.V2 = input.V8; - output.V3 = input.V24; - output.V4 = input.V4; - output.V5 = input.V20; - output.V6 = input.V12; - output.V7 = input.V28; - output.V8 = input.V2; - output.V9 = input.V18; - output.V10 = input.V10; - output.V11 = input.V26; - output.V12 = input.V6; - output.V13 = input.V22; - output.V14 = input.V14; - output.V15 = input.V30; - output.V16 = input.V1; - output.V17 = input.V17; - output.V18 = input.V9; - output.V19 = input.V25; - output.V20 = input.V5; - output.V21 = input.V21; - output.V22 = input.V13; - output.V23 = input.V29; - output.V24 = input.V3; - output.V25 = input.V19; - output.V26 = input.V11; - output.V27 = input.V27; - output.V28 = input.V7; - output.V29 = input.V23; - output.V30 = input.V15; - output.V31 = input.V31; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); - step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); - step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); - output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); - output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); - output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); - output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); - output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); - output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); - output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); - output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); - output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); - output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); - step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); - step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); - step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); - step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); - step.V16 = output.V16; - step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); - step.V31 = output.V31; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - output.V8 = step.V8; - output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); - output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); - output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); - output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); - output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); - output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); - output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); - step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); - step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); - step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); - output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); - output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); - output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); - output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); - step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); - step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); - step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); - output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); - output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); - output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs deleted file mode 100644 index e638f6a06..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the four-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Dct4Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative four-point AV1 inverse discrete cosine transform. - /// - /// The four frequency-domain coefficients. - /// The four spatial-domain residual values. - /// The four-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output[0] = input[0]; - output[1] = input[2]; - output[2] = input[1]; - output[3] = input[3]; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output.V0 = input.V0; - output.V1 = input.V2; - output.V2 = input.V1; - output.V3 = input.V3; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output.V0 = input.V0; - output.V1 = input.V2; - output.V2 = input.V1; - output.V3 = input.V3; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs deleted file mode 100644 index c532aaed1..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs +++ /dev/null @@ -1,2272 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 64-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Dct64Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 64-point AV1 inverse discrete cosine transform. - /// - /// The 64 frequency-domain coefficients. - /// The 64 spatial-domain residual values. - /// The 64-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[32]; - output[2] = input[16]; - output[3] = input[48]; - output[4] = input[8]; - output[5] = input[40]; - output[6] = input[24]; - output[7] = input[56]; - output[8] = input[4]; - output[9] = input[36]; - output[10] = input[20]; - output[11] = input[52]; - output[12] = input[12]; - output[13] = input[44]; - output[14] = input[28]; - output[15] = input[60]; - output[16] = input[2]; - output[17] = input[34]; - output[18] = input[18]; - output[19] = input[50]; - output[20] = input[10]; - output[21] = input[42]; - output[22] = input[26]; - output[23] = input[58]; - output[24] = input[6]; - output[25] = input[38]; - output[26] = input[22]; - output[27] = input[54]; - output[28] = input[14]; - output[29] = input[46]; - output[30] = input[30]; - output[31] = input[62]; - output[32] = input[1]; - output[33] = input[33]; - output[34] = input[17]; - output[35] = input[49]; - output[36] = input[9]; - output[37] = input[41]; - output[38] = input[25]; - output[39] = input[57]; - output[40] = input[5]; - output[41] = input[37]; - output[42] = input[21]; - output[43] = input[53]; - output[44] = input[13]; - output[45] = input[45]; - output[46] = input[29]; - output[47] = input[61]; - output[48] = input[3]; - output[49] = input[35]; - output[50] = input[19]; - output[51] = input[51]; - output[52] = input[11]; - output[53] = input[43]; - output[54] = input[27]; - output[55] = input[59]; - output[56] = input[7]; - output[57] = input[39]; - output[58] = input[23]; - output[59] = input[55]; - output[60] = input[15]; - output[61] = input[47]; - output[62] = input[31]; - output[63] = input[63]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); - output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); - step[63] = output[63]; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); - step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); - step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); - step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); - step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); - step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); - step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); - output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); - output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); - output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); - output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); - output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); - output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); - output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); - output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); - output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V32; - output.V2 = input.V16; - output.V3 = input.V48; - output.V4 = input.V8; - output.V5 = input.V40; - output.V6 = input.V24; - output.V7 = input.V56; - output.V8 = input.V4; - output.V9 = input.V36; - output.V10 = input.V20; - output.V11 = input.V52; - output.V12 = input.V12; - output.V13 = input.V44; - output.V14 = input.V28; - output.V15 = input.V60; - output.V16 = input.V2; - output.V17 = input.V34; - output.V18 = input.V18; - output.V19 = input.V50; - output.V20 = input.V10; - output.V21 = input.V42; - output.V22 = input.V26; - output.V23 = input.V58; - output.V24 = input.V6; - output.V25 = input.V38; - output.V26 = input.V22; - output.V27 = input.V54; - output.V28 = input.V14; - output.V29 = input.V46; - output.V30 = input.V30; - output.V31 = input.V62; - output.V32 = input.V1; - output.V33 = input.V33; - output.V34 = input.V17; - output.V35 = input.V49; - output.V36 = input.V9; - output.V37 = input.V41; - output.V38 = input.V25; - output.V39 = input.V57; - output.V40 = input.V5; - output.V41 = input.V37; - output.V42 = input.V21; - output.V43 = input.V53; - output.V44 = input.V13; - output.V45 = input.V45; - output.V46 = input.V29; - output.V47 = input.V61; - output.V48 = input.V3; - output.V49 = input.V35; - output.V50 = input.V19; - output.V51 = input.V51; - output.V52 = input.V11; - output.V53 = input.V43; - output.V54 = input.V27; - output.V55 = input.V59; - output.V56 = input.V7; - output.V57 = input.V39; - output.V58 = input.V23; - output.V59 = input.V55; - output.V60 = input.V15; - output.V61 = input.V47; - output.V62 = input.V31; - output.V63 = input.V63; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = output.V21; - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = output.V26; - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); - step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); - step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = step.V10; - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = step.V13; - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); - output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); - output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); - step.V43 = output.V43; - step.V44 = output.V44; - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); - step.V51 = output.V51; - step.V52 = output.V52; - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); - step.V63 = output.V63; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); - output.V19 = step.V19; - output.V20 = step.V20; - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); - output.V27 = step.V27; - output.V28 = step.V28; - output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = output.V41; - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); - step.V54 = output.V54; - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); - output.V22 = step.V22; - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = step.V25; - output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); - step.V44 = output.V44; - step.V45 = output.V45; - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = output.V50; - step.V51 = output.V51; - step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = step.V18; - output.V19 = step.V19; - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V28 = step.V28; - output.V29 = step.V29; - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); - step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); - step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); - step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); - step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); - step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); - step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = output.V37; - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = output.V58; - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); - output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); - output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); - output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); - output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); - output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); - output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); - output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); - output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); - output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V32; - output.V2 = input.V16; - output.V3 = input.V48; - output.V4 = input.V8; - output.V5 = input.V40; - output.V6 = input.V24; - output.V7 = input.V56; - output.V8 = input.V4; - output.V9 = input.V36; - output.V10 = input.V20; - output.V11 = input.V52; - output.V12 = input.V12; - output.V13 = input.V44; - output.V14 = input.V28; - output.V15 = input.V60; - output.V16 = input.V2; - output.V17 = input.V34; - output.V18 = input.V18; - output.V19 = input.V50; - output.V20 = input.V10; - output.V21 = input.V42; - output.V22 = input.V26; - output.V23 = input.V58; - output.V24 = input.V6; - output.V25 = input.V38; - output.V26 = input.V22; - output.V27 = input.V54; - output.V28 = input.V14; - output.V29 = input.V46; - output.V30 = input.V30; - output.V31 = input.V62; - output.V32 = input.V1; - output.V33 = input.V33; - output.V34 = input.V17; - output.V35 = input.V49; - output.V36 = input.V9; - output.V37 = input.V41; - output.V38 = input.V25; - output.V39 = input.V57; - output.V40 = input.V5; - output.V41 = input.V37; - output.V42 = input.V21; - output.V43 = input.V53; - output.V44 = input.V13; - output.V45 = input.V45; - output.V46 = input.V29; - output.V47 = input.V61; - output.V48 = input.V3; - output.V49 = input.V35; - output.V50 = input.V19; - output.V51 = input.V51; - output.V52 = input.V11; - output.V53 = input.V43; - output.V54 = input.V27; - output.V55 = input.V59; - output.V56 = input.V7; - output.V57 = input.V39; - output.V58 = input.V23; - output.V59 = input.V55; - output.V60 = input.V15; - output.V61 = input.V47; - output.V62 = input.V31; - output.V63 = input.V63; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = output.V21; - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = output.V26; - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); - step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); - step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = step.V10; - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = step.V13; - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); - output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); - output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); - step.V43 = output.V43; - step.V44 = output.V44; - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); - step.V51 = output.V51; - step.V52 = output.V52; - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); - step.V63 = output.V63; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); - output.V19 = step.V19; - output.V20 = step.V20; - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); - output.V27 = step.V27; - output.V28 = step.V28; - output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = output.V41; - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); - step.V54 = output.V54; - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); - output.V22 = step.V22; - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = step.V25; - output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); - step.V44 = output.V44; - step.V45 = output.V45; - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = output.V50; - step.V51 = output.V51; - step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = step.V18; - output.V19 = step.V19; - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V28 = step.V28; - output.V29 = step.V29; - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); - step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); - step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); - step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); - step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); - step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); - step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = output.V37; - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = output.V58; - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); - output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); - output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); - output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); - output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); - output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); - output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); - output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); - output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); - output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs deleted file mode 100644 index 5a1246eba..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the eight-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly struct Av1Dct8Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative eight-point AV1 inverse discrete cosine transform. - /// - /// The eight frequency-domain coefficients. - /// The eight spatial-domain residual values. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[4]; - output[2] = input[2]; - output[3] = input[6]; - output[4] = input[1]; - output[5] = input[5]; - output[6] = input[3]; - output[7] = input[7]; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V4; - output.V2 = input.V2; - output.V3 = input.V6; - output.V4 = input.V1; - output.V5 = input.V5; - output.V6 = input.V3; - output.V7 = input.V7; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V4; - output.V2 = input.V2; - output.V3 = input.V6; - output.V4 = input.V1; - output.V5 = input.V5; - output.V6 = input.V3; - output.V7 = input.V7; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs deleted file mode 100644 index 1f9a6c448..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the sixteen-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly struct Av1Identity16Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative sixteen-point AV1 inverse identity transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization. - for (int i = 0; i < 16; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and - // rounding sequence, matching libaom without changing the established lower-range SIMD path. - if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs deleted file mode 100644 index 416aad01b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the thirty-two-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly struct Av1Identity32Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative thirty-two-point AV1 inverse identity transform. - /// - /// The thirty-two frequency-domain coefficients. - /// The thirty-two scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization. - for (int i = 0; i < 32; i++) - { - output[i] = input[i] * 4; - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs deleted file mode 100644 index 367aa4dbf..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the four-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly struct Av1Identity4Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative four-point AV1 inverse identity transform. - /// - /// The four frequency-domain coefficients. - /// The four scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization. - for (int i = 0; i < 4; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product. - // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges. - if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs deleted file mode 100644 index 8988566b2..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the eight-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly struct Av1Identity8Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative eight-point AV1 inverse identity transform. - /// - /// The eight frequency-domain coefficients. - /// The eight scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization. - for (int i = 0; i < 8; i++) - { - output[i] = input[i] * 2; - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst16Operator.cs new file mode 100644 index 000000000..eaaeb6590 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst16Operator.cs @@ -0,0 +1,571 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Adst16Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen spatial-domain residual values. + /// The sixteen-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[15]; + output[1] = input[0]; + output[2] = input[13]; + output[3] = input[2]; + output[4] = input[11]; + output[5] = input[4]; + output[6] = input[9]; + output[7] = input[6]; + output[8] = input[7]; + output[9] = input[8]; + output[10] = input[5]; + output[11] = input[10]; + output[12] = input[3]; + output[13] = input[12]; + output[14] = input[1]; + output[15] = input[14]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); + step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); + step[12] = output[12]; + step[13] = output[13]; + step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[8]; + output[2] = step[12]; + output[3] = -step[4]; + output[4] = step[6]; + output[5] = -step[14]; + output[6] = step[10]; + output[7] = -step[2]; + output[8] = step[3]; + output[9] = -step[11]; + output[10] = step[15]; + output[11] = -step[7]; + output[12] = step[5]; + output[13] = -step[13]; + output[14] = step[9]; + output[15] = -step[1]; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst4Operator.cs new file mode 100644 index 000000000..89f890230 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst4Operator.cs @@ -0,0 +1,145 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Adst4Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform. + /// + /// The four frequency-domain coefficients. + /// The four spatial-domain residual values. + /// The stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the sine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + + // libaom widens the complete four-point factorization because the products retain their fixed-point scale + // until the final shift. The stage buffer is therefore unnecessary for this transform size. + long x0 = input[0]; + long x1 = input[1]; + long x2 = input[2]; + long x3 = input[3]; + + _ = step; + _ = stageRange; + + // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel. + if ((x0 | x1 | x2 | x3) == 0) + { + output[..4].Clear(); + return; + } + + // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3. + long s0 = sinpi[1] * x0; + long s1 = sinpi[2] * x0; + long s2 = sinpi[3] * x1; + long s3 = sinpi[4] * x2; + long s4 = sinpi[1] * x2; + long s5 = sinpi[2] * x3; + long s6 = sinpi[4] * x3; + long s7 = (x0 - x2) + x3; + + // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding. + s0 += s3; + s1 -= s4; + s3 = s2; + s2 = sinpi[3] * s7; + s0 += s5; + s1 -= s6; + x0 = s0 + s3; + x1 = s1 + s3; + x2 = s2; + x3 = (s0 + s1) - s3; + + output[0] = Av1Math.RoundShift(x0, cosBit); + output[1] = Av1Math.RoundShift(x1, cosBit); + output[2] = Av1Math.RoundShift(x2, cosBit); + output[3] = Av1Math.RoundShift(x3, cosBit); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + bool widenedRound = stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector128 x0 = input.V0; + Vector128 x1 = input.V1; + Vector128 x2 = input.V2; + Vector128 x3 = input.V3; + + // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row + // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow + // a valid Int32 fixed-point sum. + if (widenedRound) + { + output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + return; + } + + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + + _ = step; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + bool widenedRound = stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector256 x0 = input.V0; + Vector256 x1 = input.V1; + Vector256 x2 = input.V2; + Vector256 x3 = input.V3; + + if (widenedRound) + { + output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + return; + } + + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + + _ = step; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst8Operator.cs new file mode 100644 index 000000000..e9646e8e2 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Adst8Operator.cs @@ -0,0 +1,292 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Adst8Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform. + /// + /// The eight frequency-domain coefficients. + /// The eight spatial-domain residual values. + /// The eight-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[7]; + output[1] = input[0]; + output[2] = input[5]; + output[3] = input[2]; + output[4] = input[3]; + output[5] = input[4]; + output[6] = input[1]; + output[7] = input[6]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[4]; + output[2] = step[6]; + output[3] = -step[2]; + output[4] = step[3]; + output[5] = -step[7]; + output[6] = step[5]; + output[7] = -step[1]; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct16Operator.cs new file mode 100644 index 000000000..b23dcf616 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct16Operator.cs @@ -0,0 +1,478 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the 16-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Dct16Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative 16-point AV1 inverse discrete cosine transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen spatial-domain residual values. + /// The sixteen-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[8]; + output[2] = input[4]; + output[3] = input[12]; + output[4] = input[2]; + output[5] = input[10]; + output[6] = input[6]; + output[7] = input[14]; + output[8] = input[1]; + output[9] = input[9]; + output[10] = input[5]; + output[11] = input[13]; + output[12] = input[3]; + output[13] = input[11]; + output[14] = input[7]; + output[15] = input[15]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct32Operator.cs new file mode 100644 index 000000000..4f284da31 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct32Operator.cs @@ -0,0 +1,1030 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the 32-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Dct32Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative 32-point AV1 inverse discrete cosine transform. + /// + /// The 32 frequency-domain coefficients. + /// The 32 spatial-domain residual values. + /// The 32-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[16]; + output[2] = input[8]; + output[3] = input[24]; + output[4] = input[4]; + output[5] = input[20]; + output[6] = input[12]; + output[7] = input[28]; + output[8] = input[2]; + output[9] = input[18]; + output[10] = input[10]; + output[11] = input[26]; + output[12] = input[6]; + output[13] = input[22]; + output[14] = input[14]; + output[15] = input[30]; + output[16] = input[1]; + output[17] = input[17]; + output[18] = input[9]; + output[19] = input[25]; + output[20] = input[5]; + output[21] = input[21]; + output[22] = input[13]; + output[23] = input[29]; + output[24] = input[3]; + output[25] = input[19]; + output[26] = input[11]; + output[27] = input[27]; + output[28] = input[7]; + output[29] = input[23]; + output[30] = input[15]; + output[31] = input[31]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); + step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); + step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); + output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); + output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); + output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); + output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); + output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); + output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); + output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); + output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); + output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); + output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); + step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); + step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); + step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); + step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); + step[16] = output[16]; + step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); + step[19] = output[19]; + step[20] = output[20]; + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); + step[23] = output[23]; + step[24] = output[24]; + step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); + step[27] = output[27]; + step[28] = output[28]; + step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); + step[31] = output[31]; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + output[8] = step[8]; + output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); + output[11] = step[11]; + output[12] = step[12]; + output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); + output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); + output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); + output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); + output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); + output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); + output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); + step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); + step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); + step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); + step[30] = output[30]; + step[31] = output[31]; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + output[8] = step[8]; + output[9] = step[9]; + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); + output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); + output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); + output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); + output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); + step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); + step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); + step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); + output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); + output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); + output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct4Operator.cs new file mode 100644 index 000000000..149d64681 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct4Operator.cs @@ -0,0 +1,115 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the four-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Dct4Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative four-point AV1 inverse discrete cosine transform. + /// + /// The four frequency-domain coefficients. + /// The four spatial-domain residual values. + /// The four-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output[0] = input[0]; + output[1] = input[2]; + output[2] = input[1]; + output[3] = input[3]; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct64Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct64Operator.cs new file mode 100644 index 000000000..f3659699e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct64Operator.cs @@ -0,0 +1,2275 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the 64-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Dct64Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative 64-point AV1 inverse discrete cosine transform. + /// + /// The 64 frequency-domain coefficients. + /// The 64 spatial-domain residual values. + /// The 64-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[32]; + output[2] = input[16]; + output[3] = input[48]; + output[4] = input[8]; + output[5] = input[40]; + output[6] = input[24]; + output[7] = input[56]; + output[8] = input[4]; + output[9] = input[36]; + output[10] = input[20]; + output[11] = input[52]; + output[12] = input[12]; + output[13] = input[44]; + output[14] = input[28]; + output[15] = input[60]; + output[16] = input[2]; + output[17] = input[34]; + output[18] = input[18]; + output[19] = input[50]; + output[20] = input[10]; + output[21] = input[42]; + output[22] = input[26]; + output[23] = input[58]; + output[24] = input[6]; + output[25] = input[38]; + output[26] = input[22]; + output[27] = input[54]; + output[28] = input[14]; + output[29] = input[46]; + output[30] = input[30]; + output[31] = input[62]; + output[32] = input[1]; + output[33] = input[33]; + output[34] = input[17]; + output[35] = input[49]; + output[36] = input[9]; + output[37] = input[41]; + output[38] = input[25]; + output[39] = input[57]; + output[40] = input[5]; + output[41] = input[37]; + output[42] = input[21]; + output[43] = input[53]; + output[44] = input[13]; + output[45] = input[45]; + output[46] = input[29]; + output[47] = input[61]; + output[48] = input[3]; + output[49] = input[35]; + output[50] = input[19]; + output[51] = input[51]; + output[52] = input[11]; + output[53] = input[43]; + output[54] = input[27]; + output[55] = input[59]; + output[56] = input[7]; + output[57] = input[39]; + output[58] = input[23]; + output[59] = input[55]; + output[60] = input[15]; + output[61] = input[47]; + output[62] = input[31]; + output[63] = input[63]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = output[20]; + step[21] = output[21]; + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = output[26]; + step[27] = output[27]; + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); + step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); + step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = step[10]; + output[11] = step[11]; + output[12] = step[12]; + output[13] = step[13]; + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); + output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); + output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); + output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); + step[35] = output[35]; + step[36] = output[36]; + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); + step[39] = output[39]; + step[40] = output[40]; + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); + step[43] = output[43]; + step[44] = output[44]; + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); + step[47] = output[47]; + step[48] = output[48]; + step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); + step[51] = output[51]; + step[52] = output[52]; + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); + step[55] = output[55]; + step[56] = output[56]; + step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); + step[59] = output[59]; + step[60] = output[60]; + step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); + step[63] = output[63]; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); + output[19] = step[19]; + output[20] = step[20]; + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); + output[23] = step[23]; + output[24] = step[24]; + output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); + output[27] = step[27]; + output[28] = step[28]; + output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); + step[38] = output[38]; + step[39] = output[39]; + step[40] = output[40]; + step[41] = output[41]; + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); + step[54] = output[54]; + step[55] = output[55]; + step[56] = output[56]; + step[57] = output[57]; + step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); + step[62] = output[62]; + step[63] = output[63]; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); + step[44] = output[44]; + step[45] = output[45]; + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = output[50]; + step[51] = output[51]; + step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); + step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); + step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); + step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); + step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); + step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); + step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = output[36]; + step[37] = output[37]; + step[38] = output[38]; + step[39] = output[39]; + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); + step[56] = output[56]; + step[57] = output[57]; + step[58] = output[58]; + step[59] = output[59]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); + output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); + output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); + output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); + output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); + output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); + output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); + output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); + output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); + output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct8Operator.cs new file mode 100644 index 000000000..ada40dc90 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Dct8Operator.cs @@ -0,0 +1,235 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the eight-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Dct8Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative eight-point AV1 inverse discrete cosine transform. + /// + /// The eight frequency-domain coefficients. + /// The eight spatial-domain residual values. + /// The eight-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[4]; + output[2] = input[2]; + output[3] = input[6]; + output[4] = input[1]; + output[5] = input[5]; + output[6] = input[3]; + output[7] = input[7]; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity16Operator.cs new file mode 100644 index 000000000..3a06a5814 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity16Operator.cs @@ -0,0 +1,84 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the sixteen-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Identity16Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative sixteen-point AV1 inverse identity transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization. + for (int i = 0; i < 16; i++) + { + output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and + // rounding sequence, matching libaom without changing the established lower-range SIMD path. + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity32Operator.cs new file mode 100644 index 000000000..bbed35d81 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity32Operator.cs @@ -0,0 +1,68 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the thirty-two-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Identity32Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative thirty-two-point AV1 inverse identity transform. + /// + /// The thirty-two frequency-domain coefficients. + /// The thirty-two scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization. + for (int i = 0; i < 32; i++) + { + output[i] = input[i] * 4; + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity4Operator.cs new file mode 100644 index 000000000..035d29d4e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity4Operator.cs @@ -0,0 +1,84 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the four-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Identity4Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative four-point AV1 inverse identity transform. + /// + /// The four frequency-domain coefficients. + /// The four scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization. + for (int i = 0; i < 4; i++) + { + output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product. + // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges. + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity8Operator.cs new file mode 100644 index 000000000..ca9f21f76 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Identity8Operator.cs @@ -0,0 +1,68 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the eight-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal static partial class Av1Inverse2dTransformer +{ + internal readonly struct Identity8Operator : IAv1Transform1dOperator + { + /// + /// Applies the normative eight-point AV1 inverse identity transform. + /// + /// The eight frequency-domain coefficients. + /// The eight scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization. + for (int i = 0; i < 8; i++) + { + output[i] = input[i] * 2; + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Operator.cs new file mode 100644 index 000000000..579524b3c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Inverse2dTransformer.Operator.cs @@ -0,0 +1,62 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the inverse-transform operator contract. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional inverse transform. + /// + /// + /// Each overload performs the same staged fixed-point transform. Vector fields identify coefficient positions, + /// while vector lanes identify independent rows or columns. + /// + internal interface IAv1Transform1dOperator + { + /// + /// Transforms one axis when hardware vectorization is unavailable. + /// + /// The source values for the transform axis. + /// The destination values for the transform axis. + /// The fixed stage storage for the transform axis. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange); + + /// + /// Transforms four independent axes in parallel. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The fixed stage storage for four transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); + + /// + /// Transforms eight independent axes in parallel. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The fixed stage storage for eight transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); + } +} diff --git a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.Operator.cs b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.VectorOperations.cs similarity index 56% rename from src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.Operator.cs rename to src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.VectorOperations.cs index db908d0df..d2f9a2384 100644 --- a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.VectorOperations.cs @@ -3,13 +3,13 @@ using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; -using static SixLabors.ImageSharp.Formats.Heif.Components.HeifTransferVectorOperators; +using static SixLabors.ImageSharp.Formats.Heif.Components.HeifTransferVectorOperations; namespace SixLabors.ImageSharp.Formats.Heif.Components; /// /// Provides fixed-width vector overloads and shared H.273 transfer operations for HEIF color conversion. One lane -/// represents one normalized color component. Closed vector operators bind the 128-, 256-, or 512-bit implementation +/// represents one normalized color component. Closed vector operations implementations bind the 128-, 256-, or 512-bit implementation /// once per row kernel, while conditional selection evaluates piecewise transfer curves without per-lane branches. /// Inputs to logarithms and powers are bounded before evaluation because SIMD selection evaluates both branches. /// @@ -22,7 +22,7 @@ internal static partial class HeifTransferFunctions /// The nonlinear signal values. /// The corresponding linear-domain values. public static Vector128 ToLinear(CicpTransferCharacteristics transferCharacteristics, Vector128 value) - => ToLinear, Vector128Operator>(transferCharacteristics, value); + => ToLinear, Vector128Operations>(transferCharacteristics, value); /// /// Converts eight nonlinear signal values to their H.273 linear-domain values. @@ -31,7 +31,7 @@ internal static partial class HeifTransferFunctions /// The nonlinear signal values. /// The corresponding linear-domain values. public static Vector256 ToLinear(CicpTransferCharacteristics transferCharacteristics, Vector256 value) - => ToLinear, Vector256Operator>(transferCharacteristics, value); + => ToLinear, Vector256Operations>(transferCharacteristics, value); /// /// Converts sixteen nonlinear signal values to their H.273 linear-domain values. @@ -40,7 +40,7 @@ internal static partial class HeifTransferFunctions /// The nonlinear signal values. /// The corresponding linear-domain values. public static Vector512 ToLinear(CicpTransferCharacteristics transferCharacteristics, Vector512 value) - => ToLinear, Vector512Operator>(transferCharacteristics, value); + => ToLinear, Vector512Operations>(transferCharacteristics, value); /// /// Converts four linear signal values to their H.273 nonlinear-domain values. @@ -49,7 +49,7 @@ internal static partial class HeifTransferFunctions /// The linear signal values. /// The corresponding nonlinear-domain values. public static Vector128 ToGamma(CicpTransferCharacteristics transferCharacteristics, Vector128 value) - => ToGamma, Vector128Operator>(transferCharacteristics, value); + => ToGamma, Vector128Operations>(transferCharacteristics, value); /// /// Converts eight linear signal values to their H.273 nonlinear-domain values. @@ -58,7 +58,7 @@ internal static partial class HeifTransferFunctions /// The linear signal values. /// The corresponding nonlinear-domain values. public static Vector256 ToGamma(CicpTransferCharacteristics transferCharacteristics, Vector256 value) - => ToGamma, Vector256Operator>(transferCharacteristics, value); + => ToGamma, Vector256Operations>(transferCharacteristics, value); /// /// Converts sixteen linear signal values to their H.273 nonlinear-domain values. @@ -67,22 +67,22 @@ internal static partial class HeifTransferFunctions /// The linear signal values. /// The corresponding nonlinear-domain values. public static Vector512 ToGamma(CicpTransferCharacteristics transferCharacteristics, Vector512 value) - => ToGamma, Vector512Operator>(transferCharacteristics, value); + => ToGamma, Vector512Operations>(transferCharacteristics, value); /// /// Converts nonlinear signal values to their H.273 linear-domain values using the selected SIMD width. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The signaled transfer characteristics. /// The nonlinear signal values. /// The corresponding linear-domain values. - private static TVector ToLinear(CicpTransferCharacteristics transferCharacteristics, TVector value) + private static TVector ToLinear(CicpTransferCharacteristics transferCharacteristics, TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector one = TOperator.Create(1F); + TVector zero = TOperations.Create(0F); + TVector one = TOperations.Create(1F); switch (transferCharacteristics) { @@ -90,45 +90,45 @@ internal static partial class HeifTransferFunctions case CicpTransferCharacteristics.ItuRBt601_7: case CicpTransferCharacteristics.ItuRBt2020_2_10bit: case CicpTransferCharacteristics.ItuRBt2020_2_12bit: - return ToLinearBt709(value); + return ToLinearBt709(value); case CicpTransferCharacteristics.Gamma2_2: - return Power(TOperator.Min(TOperator.Max(value, zero), one), 2.2F); + return Power(TOperations.Min(TOperations.Max(value, zero), one), 2.2F); case CicpTransferCharacteristics.Gamma2_8: - return Power(TOperator.Min(TOperator.Max(value, zero), one), 2.8F); + return Power(TOperations.Min(TOperations.Max(value, zero), one), 2.8F); case CicpTransferCharacteristics.SmpteSt240: - return ToLinearSmpte240(value); + return ToLinearSmpte240(value); case CicpTransferCharacteristics.Linear: - return TOperator.Min(TOperator.Max(value, zero), one); + return TOperations.Min(TOperations.Max(value, zero), one); case CicpTransferCharacteristics.Log100: { // H.273 assigns an interval to zero for logarithmic curves. The scalar midpoint convention is // selected lane-wise after evaluating the positive branch, which keeps the hot path branchless. - TVector exponent = TOperator.Multiply(TOperator.Subtract(TOperator.Min(value, one), one), TOperator.Create(2F * 2.302585092994046F)); - TVector positive = TOperator.Exp(exponent); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), TOperator.Create(0.005F), positive); + TVector exponent = TOperations.Multiply(TOperations.Subtract(TOperations.Min(value, one), one), TOperations.Create(2F * 2.302585092994046F)); + TVector positive = TOperations.Exp(exponent); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), TOperations.Create(0.005F), positive); } case CicpTransferCharacteristics.Log100Sqrt: { - TVector exponent = TOperator.Multiply(TOperator.Subtract(TOperator.Min(value, one), one), TOperator.Create(2.5F * 2.302585092994046F)); - TVector positive = TOperator.Exp(exponent); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), TOperator.Create(0.00158113883F), positive); + TVector exponent = TOperations.Multiply(TOperations.Subtract(TOperations.Min(value, one), one), TOperations.Create(2.5F * 2.302585092994046F)); + TVector positive = TOperations.Exp(exponent); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), TOperations.Create(0.00158113883F), positive); } case CicpTransferCharacteristics.Iec61966_2_4: - return ToLinearIec61966(value); + return ToLinearIec61966(value); case CicpTransferCharacteristics.ItuRBt1361_0: - return ToLinearBt1361(value); + return ToLinearBt1361(value); case CicpTransferCharacteristics.Iec61966_2_1: - return ToLinearSrgb(value); + return ToLinearSrgb(value); case CicpTransferCharacteristics.SmpteSt2084: - return ToLinearPq(value); + return ToLinearPq(value); case CicpTransferCharacteristics.SmpteSt428_1: - return TOperator.Divide(Power(TOperator.Max(value, zero), 2.6F), TOperator.Create(Smpte428Scale)); + return TOperations.Divide(Power(TOperations.Max(value, zero), 2.6F), TOperations.Create(Smpte428Scale)); case CicpTransferCharacteristics.AribStdB67: - return ToLinearHlg(value); + return ToLinearHlg(value); default: - return ToLinearBt709(value); + return ToLinearBt709(value); } } @@ -136,16 +136,16 @@ internal static partial class HeifTransferFunctions /// Converts linear signal values to their H.273 nonlinear-domain values using the selected SIMD width. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The signaled transfer characteristics. /// The linear signal values. /// The corresponding nonlinear-domain values. - private static TVector ToGamma(CicpTransferCharacteristics transferCharacteristics, TVector value) + private static TVector ToGamma(CicpTransferCharacteristics transferCharacteristics, TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector one = TOperator.Create(1F); + TVector zero = TOperations.Create(0F); + TVector one = TOperations.Create(1F); switch (transferCharacteristics) { @@ -153,47 +153,47 @@ internal static partial class HeifTransferFunctions case CicpTransferCharacteristics.ItuRBt601_7: case CicpTransferCharacteristics.ItuRBt2020_2_10bit: case CicpTransferCharacteristics.ItuRBt2020_2_12bit: - return ToGammaBt709(value); + return ToGammaBt709(value); case CicpTransferCharacteristics.Gamma2_2: - return Power(TOperator.Min(TOperator.Max(value, zero), one), 1F / 2.2F); + return Power(TOperations.Min(TOperations.Max(value, zero), one), 1F / 2.2F); case CicpTransferCharacteristics.Gamma2_8: - return Power(TOperator.Min(TOperator.Max(value, zero), one), 1F / 2.8F); + return Power(TOperations.Min(TOperations.Max(value, zero), one), 1F / 2.8F); case CicpTransferCharacteristics.SmpteSt240: - return ToGammaSmpte240(value); + return ToGammaSmpte240(value); case CicpTransferCharacteristics.Linear: - return TOperator.Min(TOperator.Max(value, zero), one); + return TOperations.Min(TOperations.Max(value, zero), one); case CicpTransferCharacteristics.Log100: { // Clamp inactive lanes to the threshold before Log. ConditionalSelect does not short-circuit, // so this prevents negative input lanes from contaminating the vector operation with NaN values. - TVector threshold = TOperator.Create(0.01F); - TVector bounded = TOperator.Min(TOperator.Max(value, threshold), one); - TVector positive = TOperator.Add(one, TOperator.Divide(TOperator.Log(bounded), TOperator.Create(2F * 2.302585092994046F))); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, threshold), zero, positive); + TVector threshold = TOperations.Create(0.01F); + TVector bounded = TOperations.Min(TOperations.Max(value, threshold), one); + TVector positive = TOperations.Add(one, TOperations.Divide(TOperations.Log(bounded), TOperations.Create(2F * 2.302585092994046F))); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, threshold), zero, positive); } case CicpTransferCharacteristics.Log100Sqrt: { - TVector threshold = TOperator.Create(0.00316227766F); - TVector bounded = TOperator.Min(TOperator.Max(value, threshold), one); - TVector positive = TOperator.Add(one, TOperator.Divide(TOperator.Log(bounded), TOperator.Create(2.5F * 2.302585092994046F))); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, threshold), zero, positive); + TVector threshold = TOperations.Create(0.00316227766F); + TVector bounded = TOperations.Min(TOperations.Max(value, threshold), one); + TVector positive = TOperations.Add(one, TOperations.Divide(TOperations.Log(bounded), TOperations.Create(2.5F * 2.302585092994046F))); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, threshold), zero, positive); } case CicpTransferCharacteristics.Iec61966_2_4: - return ToGammaIec61966(value); + return ToGammaIec61966(value); case CicpTransferCharacteristics.ItuRBt1361_0: - return ToGammaBt1361(value); + return ToGammaBt1361(value); case CicpTransferCharacteristics.Iec61966_2_1: - return ToGammaSrgb(value); + return ToGammaSrgb(value); case CicpTransferCharacteristics.SmpteSt2084: - return ToGammaPq(value); + return ToGammaPq(value); case CicpTransferCharacteristics.SmpteSt428_1: - return Power(TOperator.Multiply(TOperator.Create(Smpte428Scale), TOperator.Max(value, zero)), 1F / 2.6F); + return Power(TOperations.Multiply(TOperations.Create(Smpte428Scale), TOperations.Max(value, zero)), 1F / 2.6F); case CicpTransferCharacteristics.AribStdB67: - return ToGammaHlg(value); + return ToGammaHlg(value); default: - return ToGammaBt709(value); + return ToGammaBt709(value); } } @@ -201,331 +201,331 @@ internal static partial class HeifTransferFunctions /// Applies the inverse BT.709-family opto-electronic transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The linear signal values. - private static TVector ToLinearBt709(TVector value) + private static TVector ToLinearBt709(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector one = TOperator.Create(1F); - TVector linear = TOperator.Divide(value, TOperator.Create(4.5F)); - TVector baseValue = TOperator.Divide(TOperator.Add(value, TOperator.Create(Bt709Alpha - 1F)), TOperator.Create(Bt709Alpha)); - TVector nonlinear = Power(TOperator.Max(baseValue, zero), 1F / 0.45F); - TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(4.5F * Bt709Beta)), linear, nonlinear); + TVector zero = TOperations.Create(0F); + TVector one = TOperations.Create(1F); + TVector linear = TOperations.Divide(value, TOperations.Create(4.5F)); + TVector baseValue = TOperations.Divide(TOperations.Add(value, TOperations.Create(Bt709Alpha - 1F)), TOperations.Create(Bt709Alpha)); + TVector nonlinear = Power(TOperations.Max(baseValue, zero), 1F / 0.45F); + TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(4.5F * Bt709Beta)), linear, nonlinear); // The comparisons deliberately mirror the scalar ordering. This preserves the H.273 lower and upper // saturation rules while allowing all lanes to execute without data-dependent branches. - TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one); - return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded); + TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one); + return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded); } /// /// Applies the BT.709-family opto-electronic transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The linear signal values. /// The nonlinear signal values. - private static TVector ToGammaBt709(TVector value) + private static TVector ToGammaBt709(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector one = TOperator.Create(1F); - TVector linear = TOperator.Multiply(value, TOperator.Create(4.5F)); - TVector nonlinear = TOperator.Subtract( - TOperator.Multiply(TOperator.Create(Bt709Alpha), Power(TOperator.Max(value, zero), 0.45F)), - TOperator.Create(Bt709Alpha - 1F)); - TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(Bt709Beta)), linear, nonlinear); - TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one); - return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded); + TVector zero = TOperations.Create(0F); + TVector one = TOperations.Create(1F); + TVector linear = TOperations.Multiply(value, TOperations.Create(4.5F)); + TVector nonlinear = TOperations.Subtract( + TOperations.Multiply(TOperations.Create(Bt709Alpha), Power(TOperations.Max(value, zero), 0.45F)), + TOperations.Create(Bt709Alpha - 1F)); + TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(Bt709Beta)), linear, nonlinear); + TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one); + return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded); } /// /// Applies the inverse SMPTE ST 240 opto-electronic transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The linear signal values. - private static TVector ToLinearSmpte240(TVector value) + private static TVector ToLinearSmpte240(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector one = TOperator.Create(1F); - TVector linear = TOperator.Divide(value, TOperator.Create(4F)); - TVector baseValue = TOperator.Divide(TOperator.Add(value, TOperator.Create(Smpte240Alpha - 1F)), TOperator.Create(Smpte240Alpha)); - TVector nonlinear = Power(TOperator.Max(baseValue, zero), 1F / 0.45F); - TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(4F * Smpte240Beta)), linear, nonlinear); - TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one); - return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded); + TVector zero = TOperations.Create(0F); + TVector one = TOperations.Create(1F); + TVector linear = TOperations.Divide(value, TOperations.Create(4F)); + TVector baseValue = TOperations.Divide(TOperations.Add(value, TOperations.Create(Smpte240Alpha - 1F)), TOperations.Create(Smpte240Alpha)); + TVector nonlinear = Power(TOperations.Max(baseValue, zero), 1F / 0.45F); + TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(4F * Smpte240Beta)), linear, nonlinear); + TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one); + return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded); } /// /// Applies the SMPTE ST 240 opto-electronic transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The linear signal values. /// The nonlinear signal values. - private static TVector ToGammaSmpte240(TVector value) + private static TVector ToGammaSmpte240(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector one = TOperator.Create(1F); - TVector linear = TOperator.Multiply(value, TOperator.Create(4F)); - TVector nonlinear = TOperator.Subtract( - TOperator.Multiply(TOperator.Create(Smpte240Alpha), Power(TOperator.Max(value, zero), 0.45F)), - TOperator.Create(Smpte240Alpha - 1F)); - TVector belowOne = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(Smpte240Beta)), linear, nonlinear); - TVector bounded = TOperator.ConditionalSelect(TOperator.LessThan(value, one), belowOne, one); - return TOperator.ConditionalSelect(TOperator.LessThan(value, zero), zero, bounded); + TVector zero = TOperations.Create(0F); + TVector one = TOperations.Create(1F); + TVector linear = TOperations.Multiply(value, TOperations.Create(4F)); + TVector nonlinear = TOperations.Subtract( + TOperations.Multiply(TOperations.Create(Smpte240Alpha), Power(TOperations.Max(value, zero), 0.45F)), + TOperations.Create(Smpte240Alpha - 1F)); + TVector belowOne = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(Smpte240Beta)), linear, nonlinear); + TVector bounded = TOperations.ConditionalSelect(TOperations.LessThan(value, one), belowOne, one); + return TOperations.ConditionalSelect(TOperations.LessThan(value, zero), zero, bounded); } /// /// Applies the inverse extended IEC 61966-2-4 transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The linear signal values. - private static TVector ToLinearIec61966(TVector value) + private static TVector ToLinearIec61966(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector negativeBase = TOperator.Divide(TOperator.Subtract(value, TOperator.Create(Bt709Alpha - 1F)), TOperator.Create(-Bt709Alpha)); - TVector negative = TOperator.Negate(Power(TOperator.Max(negativeBase, TOperator.Create(0F)), 1F / 0.45F)); - TVector linear = TOperator.Divide(value, TOperator.Create(4.5F)); - TVector positiveBase = TOperator.Divide(TOperator.Add(value, TOperator.Create(Bt709Alpha - 1F)), TOperator.Create(Bt709Alpha)); - TVector positive = Power(TOperator.Max(positiveBase, TOperator.Create(0F)), 1F / 0.45F); - TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(4.5F * Bt709Beta)), linear, positive); - return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-4.5F * Bt709Beta)), negative, centerOrPositive); + TVector negativeBase = TOperations.Divide(TOperations.Subtract(value, TOperations.Create(Bt709Alpha - 1F)), TOperations.Create(-Bt709Alpha)); + TVector negative = TOperations.Negate(Power(TOperations.Max(negativeBase, TOperations.Create(0F)), 1F / 0.45F)); + TVector linear = TOperations.Divide(value, TOperations.Create(4.5F)); + TVector positiveBase = TOperations.Divide(TOperations.Add(value, TOperations.Create(Bt709Alpha - 1F)), TOperations.Create(Bt709Alpha)); + TVector positive = Power(TOperations.Max(positiveBase, TOperations.Create(0F)), 1F / 0.45F); + TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(4.5F * Bt709Beta)), linear, positive); + return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-4.5F * Bt709Beta)), negative, centerOrPositive); } /// /// Applies the extended IEC 61966-2-4 transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The linear signal values. /// The nonlinear signal values. - private static TVector ToGammaIec61966(TVector value) + private static TVector ToGammaIec61966(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector negative = TOperator.Add( - TOperator.Negate(TOperator.Multiply( - TOperator.Create(Bt709Alpha), - Power(TOperator.Max(TOperator.Negate(value), zero), 0.45F))), - TOperator.Create(Bt709Alpha - 1F)); - - TVector linear = TOperator.Multiply(value, TOperator.Create(4.5F)); - TVector positive = TOperator.Subtract( - TOperator.Multiply(TOperator.Create(Bt709Alpha), Power(TOperator.Max(value, zero), 0.45F)), - TOperator.Create(Bt709Alpha - 1F)); - TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(Bt709Beta)), linear, positive); - return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-Bt709Beta)), negative, centerOrPositive); + TVector zero = TOperations.Create(0F); + TVector negative = TOperations.Add( + TOperations.Negate(TOperations.Multiply( + TOperations.Create(Bt709Alpha), + Power(TOperations.Max(TOperations.Negate(value), zero), 0.45F))), + TOperations.Create(Bt709Alpha - 1F)); + + TVector linear = TOperations.Multiply(value, TOperations.Create(4.5F)); + TVector positive = TOperations.Subtract( + TOperations.Multiply(TOperations.Create(Bt709Alpha), Power(TOperations.Max(value, zero), 0.45F)), + TOperations.Create(Bt709Alpha - 1F)); + TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(Bt709Beta)), linear, positive); + return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-Bt709Beta)), negative, centerOrPositive); } /// /// Applies the inverse extended BT.1361 transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The linear signal values. - private static TVector ToLinearBt1361(TVector value) + private static TVector ToLinearBt1361(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector negativeBase = TOperator.Divide(TOperator.Subtract(value, TOperator.Create(0.02482420670236F)), TOperator.Create(-0.27482420670236F)); - TVector negative = TOperator.Divide(Power(TOperator.Max(negativeBase, zero), 1F / 0.45F), TOperator.Create(-4F)); - TVector negativeOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, zero), negative, ToLinearBt709(value)); - return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-0.25F)), TOperator.Create(-0.25F), negativeOrPositive); + TVector zero = TOperations.Create(0F); + TVector negativeBase = TOperations.Divide(TOperations.Subtract(value, TOperations.Create(0.02482420670236F)), TOperations.Create(-0.27482420670236F)); + TVector negative = TOperations.Divide(Power(TOperations.Max(negativeBase, zero), 1F / 0.45F), TOperations.Create(-4F)); + TVector negativeOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, zero), negative, ToLinearBt709(value)); + return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-0.25F)), TOperations.Create(-0.25F), negativeOrPositive); } /// /// Applies the extended BT.1361 transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The linear signal values. /// The nonlinear signal values. - private static TVector ToGammaBt1361(TVector value) + private static TVector ToGammaBt1361(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector negativePower = Power(TOperator.Max(TOperator.Multiply(TOperator.Create(-4F), value), zero), 0.45F); - TVector negative = TOperator.Add(TOperator.Multiply(TOperator.Create(-0.27482420670236F), negativePower), TOperator.Create(0.02482420670236F)); - TVector negativeOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, zero), negative, ToGammaBt709(value)); - return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-0.25F)), TOperator.Create(-0.25F), negativeOrPositive); + TVector zero = TOperations.Create(0F); + TVector negativePower = Power(TOperations.Max(TOperations.Multiply(TOperations.Create(-4F), value), zero), 0.45F); + TVector negative = TOperations.Add(TOperations.Multiply(TOperations.Create(-0.27482420670236F), negativePower), TOperations.Create(0.02482420670236F)); + TVector negativeOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, zero), negative, ToGammaBt709(value)); + return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-0.25F)), TOperations.Create(-0.25F), negativeOrPositive); } /// /// Applies the inverse extended IEC 61966-2-1 transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The linear signal values. - private static TVector ToLinearSrgb(TVector value) + private static TVector ToLinearSrgb(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector negativeBase = TOperator.Divide(TOperator.Subtract(value, TOperator.Create(SrgbAlpha - 1F)), TOperator.Create(-SrgbAlpha)); - TVector negative = TOperator.Negate(Power(TOperator.Max(negativeBase, zero), 2.4F)); - TVector linear = TOperator.Divide(value, TOperator.Create(12.92F)); - TVector positiveBase = TOperator.Divide(TOperator.Add(value, TOperator.Create(SrgbAlpha - 1F)), TOperator.Create(SrgbAlpha)); - TVector positive = Power(TOperator.Max(positiveBase, zero), 2.4F); - TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(12.92F * SrgbBeta)), linear, positive); - return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-12.92F * SrgbBeta)), negative, centerOrPositive); + TVector zero = TOperations.Create(0F); + TVector negativeBase = TOperations.Divide(TOperations.Subtract(value, TOperations.Create(SrgbAlpha - 1F)), TOperations.Create(-SrgbAlpha)); + TVector negative = TOperations.Negate(Power(TOperations.Max(negativeBase, zero), 2.4F)); + TVector linear = TOperations.Divide(value, TOperations.Create(12.92F)); + TVector positiveBase = TOperations.Divide(TOperations.Add(value, TOperations.Create(SrgbAlpha - 1F)), TOperations.Create(SrgbAlpha)); + TVector positive = Power(TOperations.Max(positiveBase, zero), 2.4F); + TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(12.92F * SrgbBeta)), linear, positive); + return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-12.92F * SrgbBeta)), negative, centerOrPositive); } /// /// Applies the extended IEC 61966-2-1 transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The linear signal values. /// The nonlinear signal values. - private static TVector ToGammaSrgb(TVector value) + private static TVector ToGammaSrgb(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector negative = TOperator.Add( - TOperator.Negate(TOperator.Multiply( - TOperator.Create(SrgbAlpha), - Power(TOperator.Max(TOperator.Negate(value), zero), 1F / 2.4F))), - TOperator.Create(SrgbAlpha - 1F)); - - TVector linear = TOperator.Multiply(value, TOperator.Create(12.92F)); - TVector positive = TOperator.Subtract( - TOperator.Multiply(TOperator.Create(SrgbAlpha), Power(TOperator.Max(value, zero), 1F / 2.4F)), - TOperator.Create(SrgbAlpha - 1F)); - TVector centerOrPositive = TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(SrgbBeta)), linear, positive); - return TOperator.ConditionalSelect(TOperator.LessThan(value, TOperator.Create(-SrgbBeta)), negative, centerOrPositive); + TVector zero = TOperations.Create(0F); + TVector negative = TOperations.Add( + TOperations.Negate(TOperations.Multiply( + TOperations.Create(SrgbAlpha), + Power(TOperations.Max(TOperations.Negate(value), zero), 1F / 2.4F))), + TOperations.Create(SrgbAlpha - 1F)); + + TVector linear = TOperations.Multiply(value, TOperations.Create(12.92F)); + TVector positive = TOperations.Subtract( + TOperations.Multiply(TOperations.Create(SrgbAlpha), Power(TOperations.Max(value, zero), 1F / 2.4F)), + TOperations.Create(SrgbAlpha - 1F)); + TVector centerOrPositive = TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(SrgbBeta)), linear, positive); + return TOperations.ConditionalSelect(TOperations.LessThan(value, TOperations.Create(-SrgbBeta)), negative, centerOrPositive); } /// /// Applies the inverse SMPTE ST 2084 perceptual-quantizer transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The normalized linear signal values. - private static TVector ToLinearPq(TVector value) + private static TVector ToLinearPq(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector nonlinearPower = Power(TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F)), 1F / PqM); - TVector numerator = TOperator.Max(TOperator.Subtract(nonlinearPower, TOperator.Create(PqC1)), zero); - TVector denominator = TOperator.Subtract(TOperator.Create(PqC2), TOperator.Multiply(TOperator.Create(PqC3), nonlinearPower)); - TVector positive = TOperator.Min(Power(TOperator.Divide(numerator, denominator), 1F / PqN), TOperator.Create(1F)); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive); + TVector zero = TOperations.Create(0F); + TVector nonlinearPower = Power(TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F)), 1F / PqM); + TVector numerator = TOperations.Max(TOperations.Subtract(nonlinearPower, TOperations.Create(PqC1)), zero); + TVector denominator = TOperations.Subtract(TOperations.Create(PqC2), TOperations.Multiply(TOperations.Create(PqC3), nonlinearPower)); + TVector positive = TOperations.Min(Power(TOperations.Divide(numerator, denominator), 1F / PqN), TOperations.Create(1F)); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive); } /// /// Applies the SMPTE ST 2084 perceptual-quantizer transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The normalized linear signal values. /// The nonlinear signal values. - private static TVector ToGammaPq(TVector value) + private static TVector ToGammaPq(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector linearPower = Power(TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F)), PqN); - TVector numerator = TOperator.Add(TOperator.Create(PqC1), TOperator.Multiply(TOperator.Create(PqC2), linearPower)); - TVector denominator = TOperator.Add(TOperator.Create(1F), TOperator.Multiply(TOperator.Create(PqC3), linearPower)); - TVector positive = TOperator.Min(Power(TOperator.Divide(numerator, denominator), PqM), TOperator.Create(1F)); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive); + TVector zero = TOperations.Create(0F); + TVector linearPower = Power(TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F)), PqN); + TVector numerator = TOperations.Add(TOperations.Create(PqC1), TOperations.Multiply(TOperations.Create(PqC2), linearPower)); + TVector denominator = TOperations.Add(TOperations.Create(1F), TOperations.Multiply(TOperations.Create(PqC3), linearPower)); + TVector positive = TOperations.Min(Power(TOperations.Divide(numerator, denominator), PqM), TOperations.Create(1F)); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive); } /// /// Applies the inverse HLG opto-electronic transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonlinear signal values. /// The normalized linear signal values. - private static TVector ToLinearHlg(TVector value) + private static TVector ToLinearHlg(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector bounded = TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F)); - TVector linear = TOperator.Divide(TOperator.Multiply(bounded, bounded), TOperator.Create(3F)); - TVector exponent = TOperator.Divide(TOperator.Subtract(bounded, TOperator.Create(HlgC)), TOperator.Create(HlgA)); - TVector logarithmic = TOperator.Divide(TOperator.Add(TOperator.Exp(exponent), TOperator.Create(HlgB)), TOperator.Create(12F)); - TVector positive = TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, TOperator.Create(0.5F)), linear, logarithmic); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive); + TVector zero = TOperations.Create(0F); + TVector bounded = TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F)); + TVector linear = TOperations.Divide(TOperations.Multiply(bounded, bounded), TOperations.Create(3F)); + TVector exponent = TOperations.Divide(TOperations.Subtract(bounded, TOperations.Create(HlgC)), TOperations.Create(HlgA)); + TVector logarithmic = TOperations.Divide(TOperations.Add(TOperations.Exp(exponent), TOperations.Create(HlgB)), TOperations.Create(12F)); + TVector positive = TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, TOperations.Create(0.5F)), linear, logarithmic); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive); } /// /// Applies the HLG opto-electronic transfer function to a SIMD vector. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The normalized linear signal values. /// The nonlinear signal values. - private static TVector ToGammaHlg(TVector value) + private static TVector ToGammaHlg(TVector value) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { - TVector zero = TOperator.Create(0F); - TVector bounded = TOperator.Min(TOperator.Max(value, zero), TOperator.Create(1F)); - TVector linear = TOperator.Sqrt(TOperator.Multiply(TOperator.Create(3F), bounded)); + TVector zero = TOperations.Create(0F); + TVector bounded = TOperations.Min(TOperations.Max(value, zero), TOperations.Create(1F)); + TVector linear = TOperations.Sqrt(TOperations.Multiply(TOperations.Create(3F), bounded)); // Clamp the logarithm input for inactive lanes. SIMD conditional selection evaluates both branches, // while the scalar definition evaluates Log only above the 1/12 transition. - TVector logarithmInput = TOperator.Max( - TOperator.Subtract(TOperator.Multiply(TOperator.Create(12F), bounded), TOperator.Create(HlgB)), - TOperator.Create(float.Epsilon)); - TVector logarithmic = TOperator.Add(TOperator.Multiply(TOperator.Create(HlgA), TOperator.Log(logarithmInput)), TOperator.Create(HlgC)); - TVector positive = TOperator.ConditionalSelect(TOperator.LessThanOrEqual(bounded, TOperator.Create(1F / 12F)), linear, logarithmic); - return TOperator.ConditionalSelect(TOperator.LessThanOrEqual(value, zero), zero, positive); + TVector logarithmInput = TOperations.Max( + TOperations.Subtract(TOperations.Multiply(TOperations.Create(12F), bounded), TOperations.Create(HlgB)), + TOperations.Create(float.Epsilon)); + TVector logarithmic = TOperations.Add(TOperations.Multiply(TOperations.Create(HlgA), TOperations.Log(logarithmInput)), TOperations.Create(HlgC)); + TVector positive = TOperations.ConditionalSelect(TOperations.LessThanOrEqual(bounded, TOperations.Create(1F / 12F)), linear, logarithmic); + return TOperations.ConditionalSelect(TOperations.LessThanOrEqual(value, zero), zero, positive); } /// /// Raises nonnegative SIMD values to a scalar exponent. /// /// The SIMD vector type. - /// The operations for the SIMD vector type. + /// The operations for the SIMD vector type. /// The nonnegative base values. /// The exponent applied to every lane. /// The exponentiated values. - private static TVector Power(TVector value, float exponent) + private static TVector Power(TVector value, float exponent) where TVector : struct - where TOperator : struct, ITransferVectorOperator + where TOperations : struct, ITransferVectorOperations { // System.Numerics.Tensors does not currently vectorize Pow. Expressing positive powers as Exp(Log(x) * y) // uses the .NET 10 cross-platform vector math kernels and keeps all transfer-function lanes in SIMD. - return TOperator.Exp(TOperator.Multiply(TOperator.Log(value), TOperator.Create(exponent))); + return TOperations.Exp(TOperations.Multiply(TOperations.Log(value), TOperations.Create(exponent))); } } /// -/// Contains the stateless vector-width operators used by the shared H.273 transfer-function formulas. +/// Contains the vector-width operations used by the shared H.273 transfer-function formulas. /// -internal static class HeifTransferVectorOperators +internal static class HeifTransferVectorOperations { /// /// Defines the lane-wise operations required by the shared H.273 SIMD formulas. /// /// The SIMD vector type. - public interface ITransferVectorOperator + public interface ITransferVectorOperations where TVector : struct { /// @@ -649,7 +649,7 @@ internal static class HeifTransferVectorOperators /// /// Maps the shared transfer-function formulas to 128-bit vector operations. /// - public readonly struct Vector128Operator : ITransferVectorOperator> + public readonly struct Vector128Operations : ITransferVectorOperations> { /// public static Vector128 Create(float value) => Vector128.Create(value); @@ -702,7 +702,7 @@ internal static class HeifTransferVectorOperators /// /// Maps the shared transfer-function formulas to 256-bit vector operations. /// - public readonly struct Vector256Operator : ITransferVectorOperator> + public readonly struct Vector256Operations : ITransferVectorOperations> { /// public static Vector256 Create(float value) => Vector256.Create(value); @@ -755,7 +755,7 @@ internal static class HeifTransferVectorOperators /// /// Maps the shared transfer-function formulas to 512-bit vector operations. /// - public readonly struct Vector512Operator : ITransferVectorOperator> + public readonly struct Vector512Operations : ITransferVectorOperations> { /// public static Vector512 Create(float value) => Vector512.Create(value); diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.HorizontalEdgeOperator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.HorizontalEdgeOperator.cs new file mode 100644 index 000000000..d2f62c337 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.HorizontalEdgeOperator.cs @@ -0,0 +1,64 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +internal static partial class HevcDeblockingFilter +{ + /// + /// Accesses four columns across a horizontal edge. + /// + private readonly struct HorizontalEdgeOperator : IEdgeOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count) + { + ref ushort source = ref picture.GetRowSpan(plane, y + distance)[x]; + if (count == 2) + { + // The packed load used by full-width segments would read two samples beyond a subsampled edge. + return Vector128.Create((int)source, Unsafe.Add(ref source, 1), 0, 0); + } + + Vector64 packed = Unsafe.As>(ref source); + return Vector128.WidenLower(Vector128.Create(packed, Vector64.Zero)).AsInt32(); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreVector( + HevcPictureBuffer picture, + HevcPlane plane, + int x, + int y, + int distance, + Vector128 value, + int count) + { + ref ushort destination = ref picture.GetRowSpan(plane, y + distance)[x]; + if (count == 4) + { + Vector64 packed = Vector128.Narrow(value, Vector128.Zero).AsUInt16().GetLower(); + Unsafe.As>(ref destination) = packed; + return; + } + + destination = (ushort)value.GetElement(0); + Unsafe.Add(ref destination, 1) = (ushort)value.GetElement(1); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index) + => picture.GetRowSpan(plane, y + distance)[x + index]; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value) + => picture.GetRowSpan(plane, y + distance)[x + index] = (ushort)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.Operator.cs new file mode 100644 index 000000000..19581b352 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.Operator.cs @@ -0,0 +1,70 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +internal static partial class HevcDeblockingFilter +{ + /// + /// Defines orientation-specific access to the four samples running along one deblocking edge segment. + /// + private interface IEdgeOperator + { + /// + /// Loads samples at one signed distance across the edge. + /// + /// The reconstructed picture. + /// The component plane. + /// The first Q-side sample X coordinate. + /// The first Q-side sample Y coordinate. + /// The signed sample distance across the edge. + /// The number of valid low lanes to load. + /// The widened samples ordered along the edge. + public static abstract Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count); + + /// + /// Stores four samples at one signed distance across the edge. + /// + /// The reconstructed picture. + /// The component plane. + /// The first Q-side sample X coordinate. + /// The first Q-side sample Y coordinate. + /// The signed sample distance across the edge. + /// The four widened samples ordered along the edge. + /// The number of low lanes to store. + public static abstract void StoreVector( + HevcPictureBuffer picture, + HevcPlane plane, + int x, + int y, + int distance, + Vector128 value, + int count); + + /// + /// Loads one scalar sample at a signed distance across and an offset along the edge. + /// + /// The reconstructed picture. + /// The component plane. + /// The first Q-side sample X coordinate. + /// The first Q-side sample Y coordinate. + /// The signed sample distance across the edge. + /// The sample offset along the edge. + /// The selected sample. + public static abstract int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index); + + /// + /// Stores one scalar sample at a signed distance across and an offset along the edge. + /// + /// The reconstructed picture. + /// The component plane. + /// The first Q-side sample X coordinate. + /// The first Q-side sample Y coordinate. + /// The signed sample distance across the edge. + /// The sample offset along the edge. + /// The filtered sample. + public static abstract void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value); + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.VerticalEdgeOperator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.VerticalEdgeOperator.cs new file mode 100644 index 000000000..e5dccfafb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.VerticalEdgeOperator.cs @@ -0,0 +1,65 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +internal static partial class HevcDeblockingFilter +{ + /// + /// Accesses four rows across a vertical edge. + /// + private readonly struct VerticalEdgeOperator : IEdgeOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count) + { + if (count == 4) + { + return Vector128.Create( + (int)picture.GetRowSpan(plane, y)[x + distance], + picture.GetRowSpan(plane, y + 1)[x + distance], + picture.GetRowSpan(plane, y + 2)[x + distance], + picture.GetRowSpan(plane, y + 3)[x + distance]); + } + + // Subsampled chroma edges contain two samples. Zeroing the unused lanes keeps the vector path within + // the plane while allowing the shared kernel to operate on both valid samples in one instruction stream. + return Vector128.Create( + (int)picture.GetRowSpan(plane, y)[x + distance], + picture.GetRowSpan(plane, y + 1)[x + distance], + 0, + 0); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreVector( + HevcPictureBuffer picture, + HevcPlane plane, + int x, + int y, + int distance, + Vector128 value, + int count) + { + for (int index = 0; index < count; index++) + { + picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value.GetElement(index); + } + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index) + => picture.GetRowSpan(plane, y + index)[x + distance]; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value) + => picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs index cb4c0e32f..39ccc31d3 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs @@ -15,69 +15,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc; /// Loads with fewer than four valid positions populate only the low lanes, which the matching store writes without /// touching samples beyond the picture boundary. /// -internal static class HevcDeblockingFilter +internal static partial class HevcDeblockingFilter { - /// - /// Defines orientation-specific access to the four samples running along one deblocking edge segment. - /// - private interface IEdgeOperator - { - /// - /// Loads samples at one signed distance across the edge. - /// - /// The reconstructed picture. - /// The component plane. - /// The first Q-side sample X coordinate. - /// The first Q-side sample Y coordinate. - /// The signed sample distance across the edge. - /// The number of valid low lanes to load. - /// The widened samples ordered along the edge. - public static abstract Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count); - - /// - /// Stores four samples at one signed distance across the edge. - /// - /// The reconstructed picture. - /// The component plane. - /// The first Q-side sample X coordinate. - /// The first Q-side sample Y coordinate. - /// The signed sample distance across the edge. - /// The four widened samples ordered along the edge. - /// The number of low lanes to store. - public static abstract void StoreVector( - HevcPictureBuffer picture, - HevcPlane plane, - int x, - int y, - int distance, - Vector128 value, - int count); - - /// - /// Loads one scalar sample at a signed distance across and an offset along the edge. - /// - /// The reconstructed picture. - /// The component plane. - /// The first Q-side sample X coordinate. - /// The first Q-side sample Y coordinate. - /// The signed sample distance across the edge. - /// The sample offset along the edge. - /// The selected sample. - public static abstract int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index); - - /// - /// Stores one scalar sample at a signed distance across and an offset along the edge. - /// - /// The reconstructed picture. - /// The component plane. - /// The first Q-side sample X coordinate. - /// The first Q-side sample Y coordinate. - /// The signed sample distance across the edge. - /// The sample offset along the edge. - /// The filtered sample. - public static abstract void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value); - } - /// /// Filters four rows crossing one vertical luma boundary. /// @@ -528,113 +467,4 @@ internal static class HevcDeblockingFilter && discontinuity < (beta >> 2) && Math.Abs(p0 - q0) < strongThreshold; } - - /// - /// Accesses four rows across a vertical edge. - /// - private readonly struct VerticalEdgeOperator : IEdgeOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count) - { - if (count == 4) - { - return Vector128.Create( - (int)picture.GetRowSpan(plane, y)[x + distance], - picture.GetRowSpan(plane, y + 1)[x + distance], - picture.GetRowSpan(plane, y + 2)[x + distance], - picture.GetRowSpan(plane, y + 3)[x + distance]); - } - - // Subsampled chroma edges contain two samples. Zeroing the unused lanes keeps the vector path within - // the plane while allowing the shared kernel to operate on both valid samples in one instruction stream. - return Vector128.Create( - (int)picture.GetRowSpan(plane, y)[x + distance], - picture.GetRowSpan(plane, y + 1)[x + distance], - 0, - 0); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreVector( - HevcPictureBuffer picture, - HevcPlane plane, - int x, - int y, - int distance, - Vector128 value, - int count) - { - for (int index = 0; index < count; index++) - { - picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value.GetElement(index); - } - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index) - => picture.GetRowSpan(plane, y + index)[x + distance]; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value) - => picture.GetRowSpan(plane, y + index)[x + distance] = (ushort)value; - } - - /// - /// Accesses four columns across a horizontal edge. - /// - private readonly struct HorizontalEdgeOperator : IEdgeOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count) - { - ref ushort source = ref picture.GetRowSpan(plane, y + distance)[x]; - if (count == 2) - { - // The packed load used by full-width segments would read two samples beyond a subsampled edge. - return Vector128.Create((int)source, Unsafe.Add(ref source, 1), 0, 0); - } - - Vector64 packed = Unsafe.As>(ref source); - return Vector128.WidenLower(Vector128.Create(packed, Vector64.Zero)).AsInt32(); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreVector( - HevcPictureBuffer picture, - HevcPlane plane, - int x, - int y, - int distance, - Vector128 value, - int count) - { - ref ushort destination = ref picture.GetRowSpan(plane, y + distance)[x]; - if (count == 4) - { - Vector64 packed = Vector128.Narrow(value, Vector128.Zero).AsUInt16().GetLower(); - Unsafe.As>(ref destination) = packed; - return; - } - - destination = (ushort)value.GetElement(0); - Unsafe.Add(ref destination, 1) = (ushort)value.GetElement(1); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LoadScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index) - => picture.GetRowSpan(plane, y + distance)[x + index]; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreScalar(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int index, int value) - => picture.GetRowSpan(plane, y + distance)[x + index] = (ushort)value; - } } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.AngularOperator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.AngularOperator.cs new file mode 100644 index 000000000..e4ea8a0f1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.AngularOperator.cs @@ -0,0 +1,346 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines angular intra-prediction arithmetic. +/// +internal static partial class HevcIntraPredictor +{ + /// + /// Implements the thirty-three directional intra-prediction modes. + /// + private readonly struct AngularOperator : IHevcIntraPredictionOperator + { + /// + public static void Predict( + ReadOnlySpan top, + ReadOnlySpan left, + Span destination, + int destinationStride, + int size, + int mode, + int bitDepth, + bool filterPredictionEdges, + Span scratch) + { + if (mode == VerticalMode) + { + PredictVertical(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges); + return; + } + + if (mode == HorizontalMode) + { + PredictHorizontal(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges); + return; + } + + bool vertical = mode >= FirstVerticalMode; + int angleMode = vertical ? mode - VerticalMode : HorizontalMode - mode; + int absoluteAngleMode = Math.Abs(angleMode); + int angle = PredictionAngles[absoluteAngleMode] * Math.Sign(angleMode); + ReadOnlySpan main = vertical ? top : left; + ReadOnlySpan side = vertical ? left : top; + Span temporaryBlock = scratch[..(size * size)]; + Span extendedReference = scratch.Slice(size * size, (4 * size) + 1); + int mainOrigin = 0; + + if (angle < 0) + { + mainOrigin = size * 2; + main[..(size + 1)].CopyTo(extendedReference[mainOrigin..]); + int inverseAngle = InversePredictionAngles[absoluteAngleMode]; + int inverseAngleSum = 128; + int minimumIndex = (size * angle) >> 5; + for (int index = -1; index > minimumIndex; index--) + { + inverseAngleSum += inverseAngle; + extendedReference[mainOrigin + index] = side[inverseAngleSum >> 8]; + } + + main = extendedReference; + } + + Span prediction = vertical ? destination : temporaryBlock; + int predictionStride = vertical ? destinationStride : size; + PredictAngularRows(main, mainOrigin, prediction, predictionStride, size, angle); + if (!vertical) + { + TransposeBlock(temporaryBlock, destination, destinationStride, size); + } + } + + /// + /// Copies the top reference into every row and optionally filters the first column. + /// + /// The top reference samples. + /// The left reference samples. + /// The destination block origin. + /// The destination row stride. + /// The square block side. + /// The reconstructed component precision. + /// Whether the vertical luma edge filter applies. + private static void PredictVertical( + ReadOnlySpan top, + ReadOnlySpan left, + Span destination, + int destinationStride, + int size, + int bitDepth, + bool filterPredictionEdges) + { + ReadOnlySpan row = top.Slice(1, size); + int maximum = (1 << bitDepth) - 1; + for (int y = 0; y < size; y++) + { + row.CopyTo(destination.Slice(y * destinationStride, size)); + if (filterPredictionEdges) + { + int sample = destination[y * destinationStride] + ((left[y + 1] - left[0]) >> 1); + destination[y * destinationStride] = (ushort)Math.Clamp(sample, 0, maximum); + } + } + } + + /// + /// Fills each row from its left reference and optionally filters the first row. + /// + /// The top reference samples. + /// The left reference samples. + /// The destination block origin. + /// The destination row stride. + /// The square block side. + /// The reconstructed component precision. + /// Whether the horizontal luma edge filter applies. + private static void PredictHorizontal( + ReadOnlySpan top, + ReadOnlySpan left, + Span destination, + int destinationStride, + int size, + int bitDepth, + bool filterPredictionEdges) + { + for (int y = 0; y < size; y++) + { + destination.Slice(y * destinationStride, size).Fill(left[y + 1]); + } + + if (!filterPredictionEdges) + { + return; + } + + int maximum = (1 << bitDepth) - 1; + for (int x = 0; x < size; x++) + { + int sample = destination[x] + ((top[x + 1] - top[0]) >> 1); + destination[x] = (ushort)Math.Clamp(sample, 0, maximum); + } + } + + /// + /// Generates a vertical-oriented angular block using contiguous SIMD interpolation within each row. + /// + /// The main reference beginning at logical index zero. + /// The span index corresponding to logical reference index zero. + /// The contiguous destination or transposition scratch block. + /// The destination row stride. + /// The square block side. + /// The signed prediction displacement in thirty-second-sample units. + private static void PredictAngularRows( + ReadOnlySpan main, + int mainOrigin, + Span destination, + int destinationStride, + int size, + int angle) + { + for (int y = 0, deltaPosition = angle; y < size; y++, deltaPosition += angle) + { + int deltaInteger = deltaPosition >> 5; + int deltaFraction = deltaPosition & 31; + int sourceOffset = mainOrigin + deltaInteger + 1; + Span row = destination.Slice(y * destinationStride, size); + if (deltaFraction == 0) + { + main.Slice(sourceOffset, size).CopyTo(row); + } + else + { + InterpolateAngularRow(main[sourceOffset..], row, deltaFraction); + } + } + } + + /// + /// Interpolates one angular prediction row between consecutive main-reference samples. + /// + /// The first main-reference sample for the row. + /// The destination prediction row. + /// The right-hand weight with a denominator of thirty-two. + private static void InterpolateAngularRow(ReadOnlySpan source, Span destination, int fraction) + { + ref ushort sourceBase = ref MemoryMarshal.GetReference(source); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + uint leftWeight = (uint)(32 - fraction); + uint rightWeight = (uint)fraction; + int i = 0; + + // Adjacent source vectors overlap by one sample, aligning each left/right reference pair in the same lane. + // Widening keeps the largest 12-bit Q5 weighted sum below the UInt32 limit before narrowing to sample storage. + if (Vector512.IsHardwareAccelerated) + { + int oneVectorFromEnd = destination.Length - Vector512.Count; + for (; i <= oneVectorFromEnd; i += Vector512.Count) + { + Vector512 left = Vector512.LoadUnsafe(ref sourceBase, (nuint)i); + Vector512 right = Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + 1)); + (Vector512 leftLow, Vector512 leftHigh) = Vector512.Widen(left); + (Vector512 rightLow, Vector512 rightHigh) = Vector512.Widen(right); + Vector512 low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector512.Create(16U)) >> 5; + Vector512 high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector512.Create(16U)) >> 5; + Vector512.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i)); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int oneVectorFromEnd = destination.Length - Vector256.Count; + for (; i <= oneVectorFromEnd; i += Vector256.Count) + { + Vector256 left = Vector256.LoadUnsafe(ref sourceBase, (nuint)i); + Vector256 right = Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + 1)); + (Vector256 leftLow, Vector256 leftHigh) = Vector256.Widen(left); + (Vector256 rightLow, Vector256 rightHigh) = Vector256.Widen(right); + Vector256 low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector256.Create(16U)) >> 5; + Vector256 high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector256.Create(16U)) >> 5; + Vector256.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i)); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = destination.Length - Vector128.Count; + for (; i <= oneVectorFromEnd; i += Vector128.Count) + { + Vector128 left = Vector128.LoadUnsafe(ref sourceBase, (nuint)i); + Vector128 right = Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + 1)); + (Vector128 leftLow, Vector128 leftHigh) = Vector128.Widen(left); + (Vector128 rightLow, Vector128 rightHigh) = Vector128.Widen(right); + Vector128 low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector128.Create(16U)) >> 5; + Vector128 high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector128.Create(16U)) >> 5; + Vector128.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i)); + } + } + + for (; i < destination.Length; i++) + { + Unsafe.Add(ref destinationBase, i) = (ushort)(((source[i] * leftWeight) + (source[i + 1] * rightWeight) + 16) >> 5); + } + } + + /// + /// Transposes a square horizontal prediction block into the reconstructed destination. + /// + /// The contiguous transposed prediction block. + /// The destination block origin. + /// The destination row stride. + /// The square block side. + private static void TransposeBlock(ReadOnlySpan source, Span destination, int destinationStride, int size) + { + if (Vector128.IsHardwareAccelerated && size >= Vector128.Count) + { + for (int y = 0; y < size; y += Vector128.Count) + { + for (int x = 0; x < size; x += Vector128.Count) + { + Transpose8x8(source, destination, destinationStride, size, x, y); + } + } + + return; + } + + for (int y = 0; y < size; y++) + { + for (int x = 0; x < size; x++) + { + destination[(x * destinationStride) + y] = source[(y * size) + x]; + } + } + } + + /// + /// Transposes one eight-by-eight tile of 16-bit prediction samples. + /// + /// The contiguous source block. + /// The destination block origin. + /// The destination row stride. + /// The contiguous source row stride. + /// The tile X coordinate in the source block. + /// The tile Y coordinate in the source block. + private static void Transpose8x8( + ReadOnlySpan source, + Span destination, + int destinationStride, + int sourceStride, + int x, + int y) + { + ref ushort sourceBase = ref MemoryMarshal.GetReference(source); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + Vector128 row0 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 0) * sourceStride) + x)).AsInt16(); + Vector128 row1 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 1) * sourceStride) + x)).AsInt16(); + Vector128 row2 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 2) * sourceStride) + x)).AsInt16(); + Vector128 row3 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 3) * sourceStride) + x)).AsInt16(); + Vector128 row4 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 4) * sourceStride) + x)).AsInt16(); + Vector128 row5 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 5) * sourceStride) + x)).AsInt16(); + Vector128 row6 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 6) * sourceStride) + x)).AsInt16(); + Vector128 row7 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 7) * sourceStride) + x)).AsInt16(); + + // Three zip stages exchange one, two, then four 16-bit coordinates. The resulting vectors are the eight + // source columns in row order, so each can be stored contiguously into one destination row. + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair1 = Vector128_.UnpackHigh(row0, row1); + Vector128 pair2 = Vector128_.UnpackLow(row2, row3); + Vector128 pair3 = Vector128_.UnpackHigh(row2, row3); + Vector128 pair4 = Vector128_.UnpackLow(row4, row5); + Vector128 pair5 = Vector128_.UnpackHigh(row4, row5); + Vector128 pair6 = Vector128_.UnpackLow(row6, row7); + Vector128 pair7 = Vector128_.UnpackHigh(row6, row7); + Vector128 quad0 = Vector128_.UnpackLow(pair0.AsInt32(), pair2.AsInt32()); + Vector128 quad1 = Vector128_.UnpackHigh(pair0.AsInt32(), pair2.AsInt32()); + Vector128 quad2 = Vector128_.UnpackLow(pair1.AsInt32(), pair3.AsInt32()); + Vector128 quad3 = Vector128_.UnpackHigh(pair1.AsInt32(), pair3.AsInt32()); + Vector128 quad4 = Vector128_.UnpackLow(pair4.AsInt32(), pair6.AsInt32()); + Vector128 quad5 = Vector128_.UnpackHigh(pair4.AsInt32(), pair6.AsInt32()); + Vector128 quad6 = Vector128_.UnpackLow(pair5.AsInt32(), pair7.AsInt32()); + Vector128 quad7 = Vector128_.UnpackHigh(pair5.AsInt32(), pair7.AsInt32()); + Vector128 column0 = Vector128_.UnpackLow(quad0.AsInt64(), quad4.AsInt64()).AsUInt16(); + Vector128 column1 = Vector128_.UnpackHigh(quad0.AsInt64(), quad4.AsInt64()).AsUInt16(); + Vector128 column2 = Vector128_.UnpackLow(quad1.AsInt64(), quad5.AsInt64()).AsUInt16(); + Vector128 column3 = Vector128_.UnpackHigh(quad1.AsInt64(), quad5.AsInt64()).AsUInt16(); + Vector128 column4 = Vector128_.UnpackLow(quad2.AsInt64(), quad6.AsInt64()).AsUInt16(); + Vector128 column5 = Vector128_.UnpackHigh(quad2.AsInt64(), quad6.AsInt64()).AsUInt16(); + Vector128 column6 = Vector128_.UnpackLow(quad3.AsInt64(), quad7.AsInt64()).AsUInt16(); + Vector128 column7 = Vector128_.UnpackHigh(quad3.AsInt64(), quad7.AsInt64()).AsUInt16(); + column0.StoreUnsafe(ref destinationBase, (nuint)(((x + 0) * destinationStride) + y)); + column1.StoreUnsafe(ref destinationBase, (nuint)(((x + 1) * destinationStride) + y)); + column2.StoreUnsafe(ref destinationBase, (nuint)(((x + 2) * destinationStride) + y)); + column3.StoreUnsafe(ref destinationBase, (nuint)(((x + 3) * destinationStride) + y)); + column4.StoreUnsafe(ref destinationBase, (nuint)(((x + 4) * destinationStride) + y)); + column5.StoreUnsafe(ref destinationBase, (nuint)(((x + 5) * destinationStride) + y)); + column6.StoreUnsafe(ref destinationBase, (nuint)(((x + 6) * destinationStride) + y)); + column7.StoreUnsafe(ref destinationBase, (nuint)(((x + 7) * destinationStride) + y)); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.DcOperator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.DcOperator.cs new file mode 100644 index 000000000..02fe02def --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.DcOperator.cs @@ -0,0 +1,108 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines DC intra-prediction arithmetic. +/// +internal static partial class HevcIntraPredictor +{ + /// + /// Implements DC prediction and its optional luma boundary filter. + /// + private readonly struct DcOperator : IHevcIntraPredictionOperator + { + /// + public static void Predict( + ReadOnlySpan top, + ReadOnlySpan left, + Span destination, + int destinationStride, + int size, + int mode, + int bitDepth, + bool filterPredictionEdges, + Span scratch) + { + uint sum = SumSamples(top.Slice(1, size)) + SumSamples(left.Slice(1, size)); + ushort dc = (ushort)((sum + (uint)size) >> (BitOperations.Log2((uint)size) + 1)); + for (int y = 0; y < size; y++) + { + destination.Slice(y * destinationStride, size).Fill(dc); + } + + if (!filterPredictionEdges) + { + return; + } + + destination[0] = (ushort)((top[1] + left[1] + (2 * dc) + 2) >> 2); + for (int x = 1; x < size; x++) + { + destination[x] = (ushort)((top[x + 1] + (3 * dc) + 2) >> 2); + } + + for (int y = 1; y < size; y++) + { + destination[y * destinationStride] = (ushort)((left[y + 1] + (3 * dc) + 2) >> 2); + } + } + + /// + /// Sums reconstructed reference samples without overflowing their 16-bit storage. + /// + /// The samples to sum. + /// The exact unsigned sum. + private static uint SumSamples(ReadOnlySpan samples) + { + ref ushort samplesBase = ref MemoryMarshal.GetReference(samples); + uint sum = 0; + int i = 0; + + // Widen before reduction because a complete 64-sample, 12-bit reference edge exceeds UInt16. The shared index + // lets narrower vectors consume only the remainder from the widest available path. + if (Vector512.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector512.Count; + for (; i <= oneVectorFromEnd; i += Vector512.Count) + { + (Vector512 low, Vector512 high) = Vector512.Widen(Vector512.LoadUnsafe(ref samplesBase, (nuint)i)); + sum += Vector512.Sum(low) + Vector512.Sum(high); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector256.Count; + for (; i <= oneVectorFromEnd; i += Vector256.Count) + { + (Vector256 low, Vector256 high) = Vector256.Widen(Vector256.LoadUnsafe(ref samplesBase, (nuint)i)); + sum += Vector256.Sum(low) + Vector256.Sum(high); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector128.Count; + for (; i <= oneVectorFromEnd; i += Vector128.Count) + { + (Vector128 low, Vector128 high) = Vector128.Widen(Vector128.LoadUnsafe(ref samplesBase, (nuint)i)); + sum += Vector128.Sum(low) + Vector128.Sum(high); + } + } + + for (; i < samples.Length; i++) + { + sum += Unsafe.Add(ref samplesBase, i); + } + + return sum; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operations.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operations.cs deleted file mode 100644 index c75c17b0a..000000000 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operations.cs +++ /dev/null @@ -1,431 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Common.Helpers; - -namespace SixLabors.ImageSharp.Formats.Heif.Hevc; - -/// -/// Provides shared SIMD operations used by the closed prediction operators. Each lane represents one output column; -/// planar and angular interpolation widen 16-bit references before their Q5 weighted sums, then narrow only after the -/// normative rounding shift. Horizontal prediction reuses the vertical row kernel through a caller-owned contiguous -/// block and an eight-by-eight transpose, keeping the arithmetic identical without gathering strided destination rows. -/// -internal static partial class HevcIntraPredictor -{ - /// - /// Calculates one 512-bit half of a planar prediction row. - /// - /// The top reference samples. - /// The zero-based X coordinates. - /// The left reference sample for the row. - /// The top-right reference sample. - /// The bottom-left reference sample. - /// The top-reference weight. - /// The bottom-left-reference weight. - /// The square block side. - /// The division rounding constant. - /// The division shift. - /// The predicted samples as widened lanes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 CalculatePlanarVector( - Vector512 top, - Vector512 indices, - uint left, - uint topRight, - uint bottomLeft, - uint topWeight, - uint bottomWeight, - uint size, - uint rounding, - int shift) - { - Vector512 horizontal = ((Vector512.Create(size - 1) - indices) * left) + ((indices + Vector512.One) * topRight); - Vector512 vertical = (top * topWeight) + Vector512.Create(bottomLeft * bottomWeight); - return (horizontal + vertical + Vector512.Create(rounding)) >> shift; - } - - /// - /// Calculates one 256-bit half of a planar prediction row. - /// - /// The top reference samples. - /// The zero-based X coordinates. - /// The left reference sample for the row. - /// The top-right reference sample. - /// The bottom-left reference sample. - /// The top-reference weight. - /// The bottom-left-reference weight. - /// The square block side. - /// The division rounding constant. - /// The division shift. - /// The predicted samples as widened lanes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 CalculatePlanarVector( - Vector256 top, - Vector256 indices, - uint left, - uint topRight, - uint bottomLeft, - uint topWeight, - uint bottomWeight, - uint size, - uint rounding, - int shift) - { - Vector256 horizontal = ((Vector256.Create(size - 1) - indices) * left) + ((indices + Vector256.One) * topRight); - Vector256 vertical = (top * topWeight) + Vector256.Create(bottomLeft * bottomWeight); - return (horizontal + vertical + Vector256.Create(rounding)) >> shift; - } - - /// - /// Calculates one 128-bit half of a planar prediction row. - /// - /// The top reference samples. - /// The zero-based X coordinates. - /// The left reference sample for the row. - /// The top-right reference sample. - /// The bottom-left reference sample. - /// The top-reference weight. - /// The bottom-left-reference weight. - /// The square block side. - /// The division rounding constant. - /// The division shift. - /// The predicted samples as widened lanes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 CalculatePlanarVector( - Vector128 top, - Vector128 indices, - uint left, - uint topRight, - uint bottomLeft, - uint topWeight, - uint bottomWeight, - uint size, - uint rounding, - int shift) - { - Vector128 horizontal = ((Vector128.Create(size - 1) - indices) * left) + ((indices + Vector128.One) * topRight); - Vector128 vertical = (top * topWeight) + Vector128.Create(bottomLeft * bottomWeight); - return (horizontal + vertical + Vector128.Create(rounding)) >> shift; - } - - /// - /// Sums reconstructed reference samples without overflowing their 16-bit storage. - /// - /// The samples to sum. - /// The exact unsigned sum. - private static uint SumSamples(ReadOnlySpan samples) - { - ref ushort samplesBase = ref MemoryMarshal.GetReference(samples); - uint sum = 0; - int i = 0; - - // Widen before reduction because a complete 64-sample, 12-bit reference edge exceeds UInt16. The shared index - // lets narrower vectors consume only the remainder from the widest available path. - if (Vector512.IsHardwareAccelerated) - { - int oneVectorFromEnd = samples.Length - Vector512.Count; - for (; i <= oneVectorFromEnd; i += Vector512.Count) - { - (Vector512 low, Vector512 high) = Vector512.Widen(Vector512.LoadUnsafe(ref samplesBase, (nuint)i)); - sum += Vector512.Sum(low) + Vector512.Sum(high); - } - } - - if (Vector256.IsHardwareAccelerated) - { - int oneVectorFromEnd = samples.Length - Vector256.Count; - for (; i <= oneVectorFromEnd; i += Vector256.Count) - { - (Vector256 low, Vector256 high) = Vector256.Widen(Vector256.LoadUnsafe(ref samplesBase, (nuint)i)); - sum += Vector256.Sum(low) + Vector256.Sum(high); - } - } - - if (Vector128.IsHardwareAccelerated) - { - int oneVectorFromEnd = samples.Length - Vector128.Count; - for (; i <= oneVectorFromEnd; i += Vector128.Count) - { - (Vector128 low, Vector128 high) = Vector128.Widen(Vector128.LoadUnsafe(ref samplesBase, (nuint)i)); - sum += Vector128.Sum(low) + Vector128.Sum(high); - } - } - - for (; i < samples.Length; i++) - { - sum += Unsafe.Add(ref samplesBase, i); - } - - return sum; - } - - /// - /// Copies the top reference into every row and optionally filters the first column. - /// - /// The top reference samples. - /// The left reference samples. - /// The destination block origin. - /// The destination row stride. - /// The square block side. - /// The reconstructed component precision. - /// Whether the vertical luma edge filter applies. - private static void PredictVertical( - ReadOnlySpan top, - ReadOnlySpan left, - Span destination, - int destinationStride, - int size, - int bitDepth, - bool filterPredictionEdges) - { - ReadOnlySpan row = top.Slice(1, size); - int maximum = (1 << bitDepth) - 1; - for (int y = 0; y < size; y++) - { - row.CopyTo(destination.Slice(y * destinationStride, size)); - if (filterPredictionEdges) - { - int sample = destination[y * destinationStride] + ((left[y + 1] - left[0]) >> 1); - destination[y * destinationStride] = (ushort)Math.Clamp(sample, 0, maximum); - } - } - } - - /// - /// Fills each row from its left reference and optionally filters the first row. - /// - /// The top reference samples. - /// The left reference samples. - /// The destination block origin. - /// The destination row stride. - /// The square block side. - /// The reconstructed component precision. - /// Whether the horizontal luma edge filter applies. - private static void PredictHorizontal( - ReadOnlySpan top, - ReadOnlySpan left, - Span destination, - int destinationStride, - int size, - int bitDepth, - bool filterPredictionEdges) - { - for (int y = 0; y < size; y++) - { - destination.Slice(y * destinationStride, size).Fill(left[y + 1]); - } - - if (!filterPredictionEdges) - { - return; - } - - int maximum = (1 << bitDepth) - 1; - for (int x = 0; x < size; x++) - { - int sample = destination[x] + ((top[x + 1] - top[0]) >> 1); - destination[x] = (ushort)Math.Clamp(sample, 0, maximum); - } - } - - /// - /// Generates a vertical-oriented angular block using contiguous SIMD interpolation within each row. - /// - /// The main reference beginning at logical index zero. - /// The span index corresponding to logical reference index zero. - /// The contiguous destination or transposition scratch block. - /// The destination row stride. - /// The square block side. - /// The signed prediction displacement in thirty-second-sample units. - private static void PredictAngularRows( - ReadOnlySpan main, - int mainOrigin, - Span destination, - int destinationStride, - int size, - int angle) - { - for (int y = 0, deltaPosition = angle; y < size; y++, deltaPosition += angle) - { - int deltaInteger = deltaPosition >> 5; - int deltaFraction = deltaPosition & 31; - int sourceOffset = mainOrigin + deltaInteger + 1; - Span row = destination.Slice(y * destinationStride, size); - if (deltaFraction == 0) - { - main.Slice(sourceOffset, size).CopyTo(row); - } - else - { - InterpolateAngularRow(main[sourceOffset..], row, deltaFraction); - } - } - } - - /// - /// Interpolates one angular prediction row between consecutive main-reference samples. - /// - /// The first main-reference sample for the row. - /// The destination prediction row. - /// The right-hand weight with a denominator of thirty-two. - private static void InterpolateAngularRow(ReadOnlySpan source, Span destination, int fraction) - { - ref ushort sourceBase = ref MemoryMarshal.GetReference(source); - ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); - uint leftWeight = (uint)(32 - fraction); - uint rightWeight = (uint)fraction; - int i = 0; - - // Adjacent source vectors overlap by one sample, aligning each left/right reference pair in the same lane. - // Widening keeps the largest 12-bit Q5 weighted sum below the UInt32 limit before narrowing to sample storage. - if (Vector512.IsHardwareAccelerated) - { - int oneVectorFromEnd = destination.Length - Vector512.Count; - for (; i <= oneVectorFromEnd; i += Vector512.Count) - { - Vector512 left = Vector512.LoadUnsafe(ref sourceBase, (nuint)i); - Vector512 right = Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + 1)); - (Vector512 leftLow, Vector512 leftHigh) = Vector512.Widen(left); - (Vector512 rightLow, Vector512 rightHigh) = Vector512.Widen(right); - Vector512 low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector512.Create(16U)) >> 5; - Vector512 high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector512.Create(16U)) >> 5; - Vector512.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i)); - } - } - - if (Vector256.IsHardwareAccelerated) - { - int oneVectorFromEnd = destination.Length - Vector256.Count; - for (; i <= oneVectorFromEnd; i += Vector256.Count) - { - Vector256 left = Vector256.LoadUnsafe(ref sourceBase, (nuint)i); - Vector256 right = Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + 1)); - (Vector256 leftLow, Vector256 leftHigh) = Vector256.Widen(left); - (Vector256 rightLow, Vector256 rightHigh) = Vector256.Widen(right); - Vector256 low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector256.Create(16U)) >> 5; - Vector256 high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector256.Create(16U)) >> 5; - Vector256.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i)); - } - } - - if (Vector128.IsHardwareAccelerated) - { - int oneVectorFromEnd = destination.Length - Vector128.Count; - for (; i <= oneVectorFromEnd; i += Vector128.Count) - { - Vector128 left = Vector128.LoadUnsafe(ref sourceBase, (nuint)i); - Vector128 right = Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + 1)); - (Vector128 leftLow, Vector128 leftHigh) = Vector128.Widen(left); - (Vector128 rightLow, Vector128 rightHigh) = Vector128.Widen(right); - Vector128 low = ((leftLow * leftWeight) + (rightLow * rightWeight) + Vector128.Create(16U)) >> 5; - Vector128 high = ((leftHigh * leftWeight) + (rightHigh * rightWeight) + Vector128.Create(16U)) >> 5; - Vector128.Narrow(low, high).StoreUnsafe(ref Unsafe.Add(ref destinationBase, i)); - } - } - - for (; i < destination.Length; i++) - { - Unsafe.Add(ref destinationBase, i) = (ushort)(((source[i] * leftWeight) + (source[i + 1] * rightWeight) + 16) >> 5); - } - } - - /// - /// Transposes a square horizontal prediction block into the reconstructed destination. - /// - /// The contiguous transposed prediction block. - /// The destination block origin. - /// The destination row stride. - /// The square block side. - private static void TransposeBlock(ReadOnlySpan source, Span destination, int destinationStride, int size) - { - if (Vector128.IsHardwareAccelerated && size >= Vector128.Count) - { - for (int y = 0; y < size; y += Vector128.Count) - { - for (int x = 0; x < size; x += Vector128.Count) - { - Transpose8x8(source, destination, destinationStride, size, x, y); - } - } - - return; - } - - for (int y = 0; y < size; y++) - { - for (int x = 0; x < size; x++) - { - destination[(x * destinationStride) + y] = source[(y * size) + x]; - } - } - } - - /// - /// Transposes one eight-by-eight tile of 16-bit prediction samples. - /// - /// The contiguous source block. - /// The destination block origin. - /// The destination row stride. - /// The contiguous source row stride. - /// The tile X coordinate in the source block. - /// The tile Y coordinate in the source block. - private static void Transpose8x8( - ReadOnlySpan source, - Span destination, - int destinationStride, - int sourceStride, - int x, - int y) - { - ref ushort sourceBase = ref MemoryMarshal.GetReference(source); - ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); - Vector128 row0 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 0) * sourceStride) + x)).AsInt16(); - Vector128 row1 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 1) * sourceStride) + x)).AsInt16(); - Vector128 row2 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 2) * sourceStride) + x)).AsInt16(); - Vector128 row3 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 3) * sourceStride) + x)).AsInt16(); - Vector128 row4 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 4) * sourceStride) + x)).AsInt16(); - Vector128 row5 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 5) * sourceStride) + x)).AsInt16(); - Vector128 row6 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 6) * sourceStride) + x)).AsInt16(); - Vector128 row7 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 7) * sourceStride) + x)).AsInt16(); - - // Three zip stages exchange one, two, then four 16-bit coordinates. The resulting vectors are the eight - // source columns in row order, so each can be stored contiguously into one destination row. - Vector128 pair0 = Vector128_.UnpackLow(row0, row1); - Vector128 pair1 = Vector128_.UnpackHigh(row0, row1); - Vector128 pair2 = Vector128_.UnpackLow(row2, row3); - Vector128 pair3 = Vector128_.UnpackHigh(row2, row3); - Vector128 pair4 = Vector128_.UnpackLow(row4, row5); - Vector128 pair5 = Vector128_.UnpackHigh(row4, row5); - Vector128 pair6 = Vector128_.UnpackLow(row6, row7); - Vector128 pair7 = Vector128_.UnpackHigh(row6, row7); - Vector128 quad0 = Vector128_.UnpackLow(pair0.AsInt32(), pair2.AsInt32()); - Vector128 quad1 = Vector128_.UnpackHigh(pair0.AsInt32(), pair2.AsInt32()); - Vector128 quad2 = Vector128_.UnpackLow(pair1.AsInt32(), pair3.AsInt32()); - Vector128 quad3 = Vector128_.UnpackHigh(pair1.AsInt32(), pair3.AsInt32()); - Vector128 quad4 = Vector128_.UnpackLow(pair4.AsInt32(), pair6.AsInt32()); - Vector128 quad5 = Vector128_.UnpackHigh(pair4.AsInt32(), pair6.AsInt32()); - Vector128 quad6 = Vector128_.UnpackLow(pair5.AsInt32(), pair7.AsInt32()); - Vector128 quad7 = Vector128_.UnpackHigh(pair5.AsInt32(), pair7.AsInt32()); - Vector128 column0 = Vector128_.UnpackLow(quad0.AsInt64(), quad4.AsInt64()).AsUInt16(); - Vector128 column1 = Vector128_.UnpackHigh(quad0.AsInt64(), quad4.AsInt64()).AsUInt16(); - Vector128 column2 = Vector128_.UnpackLow(quad1.AsInt64(), quad5.AsInt64()).AsUInt16(); - Vector128 column3 = Vector128_.UnpackHigh(quad1.AsInt64(), quad5.AsInt64()).AsUInt16(); - Vector128 column4 = Vector128_.UnpackLow(quad2.AsInt64(), quad6.AsInt64()).AsUInt16(); - Vector128 column5 = Vector128_.UnpackHigh(quad2.AsInt64(), quad6.AsInt64()).AsUInt16(); - Vector128 column6 = Vector128_.UnpackLow(quad3.AsInt64(), quad7.AsInt64()).AsUInt16(); - Vector128 column7 = Vector128_.UnpackHigh(quad3.AsInt64(), quad7.AsInt64()).AsUInt16(); - column0.StoreUnsafe(ref destinationBase, (nuint)(((x + 0) * destinationStride) + y)); - column1.StoreUnsafe(ref destinationBase, (nuint)(((x + 1) * destinationStride) + y)); - column2.StoreUnsafe(ref destinationBase, (nuint)(((x + 2) * destinationStride) + y)); - column3.StoreUnsafe(ref destinationBase, (nuint)(((x + 3) * destinationStride) + y)); - column4.StoreUnsafe(ref destinationBase, (nuint)(((x + 4) * destinationStride) + y)); - column5.StoreUnsafe(ref destinationBase, (nuint)(((x + 5) * destinationStride) + y)); - column6.StoreUnsafe(ref destinationBase, (nuint)(((x + 6) * destinationStride) + y)); - column7.StoreUnsafe(ref destinationBase, (nuint)(((x + 7) * destinationStride) + y)); - } -} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operator.cs new file mode 100644 index 000000000..bf9a712c6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operator.cs @@ -0,0 +1,39 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the HEVC intra-prediction operator contract. +/// +internal static partial class HevcIntraPredictor +{ + /// + /// Defines one closed intra-prediction operation selected by the decoded mode. + /// + private interface IHevcIntraPredictionOperator + { + /// + /// Reconstructs one square prediction block. + /// + /// The top-left, top, and top-right reference samples. + /// The top-left, left, and below-left reference samples. + /// The destination buffer beginning at the block origin. + /// The destination row stride in samples. + /// The square block side in samples. + /// The decoded prediction mode. + /// The reconstructed component precision. + /// Whether the luma edge filter applies to the selected block. + /// The caller-owned block and extended-reference scratch space. + public static abstract void Predict( + ReadOnlySpan top, + ReadOnlySpan left, + Span destination, + int destinationStride, + int size, + int mode, + int bitDepth, + bool filterPredictionEdges, + Span scratch); + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.PlanarOperator.cs similarity index 60% rename from src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs rename to src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.PlanarOperator.cs index 05185bed9..951dbb40e 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.PlanarOperator.cs @@ -9,14 +9,14 @@ using System.Runtime.Intrinsics; namespace SixLabors.ImageSharp.Formats.Heif.Hevc; /// -/// Provides the closed planar, DC, and angular prediction operators. +/// Defines planar intra-prediction arithmetic. /// internal static partial class HevcIntraPredictor { /// /// Implements planar interpolation between the top, left, bottom-left, and top-right references. /// - private readonly struct PlanarPredictionOperator : IHevcIntraPredictionOperator + private readonly struct PlanarOperator : IHevcIntraPredictionOperator { /// public static void Predict( @@ -171,112 +171,101 @@ internal static partial class HevcIntraPredictor } } } - } - /// - /// Implements DC prediction and its optional luma boundary filter. - /// - private readonly struct DcPredictionOperator : IHevcIntraPredictionOperator - { - /// - public static void Predict( - ReadOnlySpan top, - ReadOnlySpan left, - Span destination, - int destinationStride, - int size, - int mode, - int bitDepth, - bool filterPredictionEdges, - Span scratch) + /// + /// Calculates one 512-bit half of a planar prediction row. + /// + /// The top reference samples. + /// The zero-based X coordinates. + /// The left reference sample for the row. + /// The top-right reference sample. + /// The bottom-left reference sample. + /// The top-reference weight. + /// The bottom-left-reference weight. + /// The square block side. + /// The division rounding constant. + /// The division shift. + /// The predicted samples as widened lanes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CalculatePlanarVector( + Vector512 top, + Vector512 indices, + uint left, + uint topRight, + uint bottomLeft, + uint topWeight, + uint bottomWeight, + uint size, + uint rounding, + int shift) { - uint sum = SumSamples(top.Slice(1, size)) + SumSamples(left.Slice(1, size)); - ushort dc = (ushort)((sum + (uint)size) >> (BitOperations.Log2((uint)size) + 1)); - for (int y = 0; y < size; y++) - { - destination.Slice(y * destinationStride, size).Fill(dc); - } - - if (!filterPredictionEdges) - { - return; - } - - destination[0] = (ushort)((top[1] + left[1] + (2 * dc) + 2) >> 2); - for (int x = 1; x < size; x++) - { - destination[x] = (ushort)((top[x + 1] + (3 * dc) + 2) >> 2); - } - - for (int y = 1; y < size; y++) - { - destination[y * destinationStride] = (ushort)((left[y + 1] + (3 * dc) + 2) >> 2); - } + Vector512 horizontal = ((Vector512.Create(size - 1) - indices) * left) + ((indices + Vector512.One) * topRight); + Vector512 vertical = (top * topWeight) + Vector512.Create(bottomLeft * bottomWeight); + return (horizontal + vertical + Vector512.Create(rounding)) >> shift; } - } - /// - /// Implements the thirty-three directional intra-prediction modes. - /// - private readonly struct AngularPredictionOperator : IHevcIntraPredictionOperator - { - /// - public static void Predict( - ReadOnlySpan top, - ReadOnlySpan left, - Span destination, - int destinationStride, - int size, - int mode, - int bitDepth, - bool filterPredictionEdges, - Span scratch) + /// + /// Calculates one 256-bit half of a planar prediction row. + /// + /// The top reference samples. + /// The zero-based X coordinates. + /// The left reference sample for the row. + /// The top-right reference sample. + /// The bottom-left reference sample. + /// The top-reference weight. + /// The bottom-left-reference weight. + /// The square block side. + /// The division rounding constant. + /// The division shift. + /// The predicted samples as widened lanes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 CalculatePlanarVector( + Vector256 top, + Vector256 indices, + uint left, + uint topRight, + uint bottomLeft, + uint topWeight, + uint bottomWeight, + uint size, + uint rounding, + int shift) { - if (mode == VerticalMode) - { - PredictVertical(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges); - return; - } - - if (mode == HorizontalMode) - { - PredictHorizontal(top, left, destination, destinationStride, size, bitDepth, filterPredictionEdges); - return; - } - - bool vertical = mode >= FirstVerticalMode; - int angleMode = vertical ? mode - VerticalMode : HorizontalMode - mode; - int absoluteAngleMode = Math.Abs(angleMode); - int angle = PredictionAngles[absoluteAngleMode] * Math.Sign(angleMode); - ReadOnlySpan main = vertical ? top : left; - ReadOnlySpan side = vertical ? left : top; - Span temporaryBlock = scratch[..(size * size)]; - Span extendedReference = scratch.Slice(size * size, (4 * size) + 1); - int mainOrigin = 0; - - if (angle < 0) - { - mainOrigin = size * 2; - main[..(size + 1)].CopyTo(extendedReference[mainOrigin..]); - int inverseAngle = InversePredictionAngles[absoluteAngleMode]; - int inverseAngleSum = 128; - int minimumIndex = (size * angle) >> 5; - for (int index = -1; index > minimumIndex; index--) - { - inverseAngleSum += inverseAngle; - extendedReference[mainOrigin + index] = side[inverseAngleSum >> 8]; - } - - main = extendedReference; - } + Vector256 horizontal = ((Vector256.Create(size - 1) - indices) * left) + ((indices + Vector256.One) * topRight); + Vector256 vertical = (top * topWeight) + Vector256.Create(bottomLeft * bottomWeight); + return (horizontal + vertical + Vector256.Create(rounding)) >> shift; + } - Span prediction = vertical ? destination : temporaryBlock; - int predictionStride = vertical ? destinationStride : size; - PredictAngularRows(main, mainOrigin, prediction, predictionStride, size, angle); - if (!vertical) - { - TransposeBlock(temporaryBlock, destination, destinationStride, size); - } + /// + /// Calculates one 128-bit half of a planar prediction row. + /// + /// The top reference samples. + /// The zero-based X coordinates. + /// The left reference sample for the row. + /// The top-right reference sample. + /// The bottom-left reference sample. + /// The top-reference weight. + /// The bottom-left-reference weight. + /// The square block side. + /// The division rounding constant. + /// The division shift. + /// The predicted samples as widened lanes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CalculatePlanarVector( + Vector128 top, + Vector128 indices, + uint left, + uint topRight, + uint bottomLeft, + uint topWeight, + uint bottomWeight, + uint size, + uint rounding, + int shift) + { + Vector128 horizontal = ((Vector128.Create(size - 1) - indices) * left) + ((indices + Vector128.One) * topRight); + Vector128 vertical = (top * topWeight) + Vector128.Create(bottomLeft * bottomWeight); + return (horizontal + vertical + Vector128.Create(rounding)) >> shift; } } } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs index 81d300457..11cc57074 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs @@ -50,37 +50,6 @@ internal static partial class HevcIntraPredictor /// private const int MaximumBlockSize = 32; - /// - /// Defines one closed intra-prediction operation selected by the decoded mode. - /// - /// The implementing operator type. - private interface IHevcIntraPredictionOperator - where TOperator : struct, IHevcIntraPredictionOperator - { - /// - /// Reconstructs one square prediction block. - /// - /// The top-left, top, and top-right reference samples. - /// The top-left, left, and below-left reference samples. - /// The destination buffer beginning at the block origin. - /// The destination row stride in samples. - /// The square block side in samples. - /// The decoded prediction mode. - /// The reconstructed component precision. - /// Whether the luma edge filter applies to the selected block. - /// The caller-owned block and extended-reference scratch space. - public static abstract void Predict( - ReadOnlySpan top, - ReadOnlySpan left, - Span destination, - int destinationStride, - int size, - int mode, - int bitDepth, - bool filterPredictionEdges, - Span scratch); - } - /// /// Gets the angle selected by each absolute angular-mode displacement. /// @@ -133,7 +102,7 @@ internal static partial class HevcIntraPredictor switch (mode) { case PlanarMode: - Predict( + Predict( top, left, destination, @@ -145,7 +114,7 @@ internal static partial class HevcIntraPredictor scratch); break; case DcMode: - Predict( + Predict( top, left, destination, @@ -157,7 +126,7 @@ internal static partial class HevcIntraPredictor scratch); break; default: - Predict( + Predict( top, left, destination, @@ -243,7 +212,7 @@ internal static partial class HevcIntraPredictor int bitDepth, bool filterPredictionEdges, Span scratch) - where TOperator : struct, IHevcIntraPredictionOperator + where TOperator : struct, IHevcIntraPredictionOperator => TOperator.Predict( top, left, diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine16Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine16Operator.cs new file mode 100644 index 000000000..e93d20ec1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine16Operator.cs @@ -0,0 +1,49 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the sixteen-point inverse discrete cosine transform operator. +/// +internal static partial class HevcInverseTransformer +{ + /// + /// Implements the sixteen-point inverse discrete cosine transform. + /// + private readonly struct DiscreteCosine16Operator : IHevcInverseTransformOperator + { + /// + public static int Size => 16; + + /// + public static bool UsesButterfly => true; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetCoefficient(int frequency, int position) + { + if (frequency == 0) + { + return 64; + } + + int angle = ((2 * position) + 1) * frequency * 2; + angle &= 127; + if (angle > 64) + { + angle = 128 - angle; + } + + // The second quadrant reuses the first-quadrant magnitude with a negative sign. + if (angle > 32) + { + return -DiscreteCosineMagnitudes[64 - angle]; + } + + return DiscreteCosineMagnitudes[angle]; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine32Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine32Operator.cs new file mode 100644 index 000000000..dc1277890 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine32Operator.cs @@ -0,0 +1,49 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the thirty-two-point inverse discrete cosine transform operator. +/// +internal static partial class HevcInverseTransformer +{ + /// + /// Implements the thirty-two-point inverse discrete cosine transform. + /// + private readonly struct DiscreteCosine32Operator : IHevcInverseTransformOperator + { + /// + public static int Size => 32; + + /// + public static bool UsesButterfly => true; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetCoefficient(int frequency, int position) + { + if (frequency == 0) + { + return 64; + } + + int angle = ((2 * position) + 1) * frequency * 1; + angle &= 127; + if (angle > 64) + { + angle = 128 - angle; + } + + // The second quadrant reuses the first-quadrant magnitude with a negative sign. + if (angle > 32) + { + return -DiscreteCosineMagnitudes[64 - angle]; + } + + return DiscreteCosineMagnitudes[angle]; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine4Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine4Operator.cs new file mode 100644 index 000000000..7bd767cc1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine4Operator.cs @@ -0,0 +1,49 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the four-point inverse discrete cosine transform operator. +/// +internal static partial class HevcInverseTransformer +{ + /// + /// Implements the four-point inverse discrete cosine transform. + /// + private readonly struct DiscreteCosine4Operator : IHevcInverseTransformOperator + { + /// + public static int Size => 4; + + /// + public static bool UsesButterfly => true; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetCoefficient(int frequency, int position) + { + if (frequency == 0) + { + return 64; + } + + int angle = ((2 * position) + 1) * frequency * 8; + angle &= 127; + if (angle > 64) + { + angle = 128 - angle; + } + + // The second quadrant reuses the first-quadrant magnitude with a negative sign. + if (angle > 32) + { + return -DiscreteCosineMagnitudes[64 - angle]; + } + + return DiscreteCosineMagnitudes[angle]; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine8Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine8Operator.cs new file mode 100644 index 000000000..7137d7843 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteCosine8Operator.cs @@ -0,0 +1,49 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the eight-point inverse discrete cosine transform operator. +/// +internal static partial class HevcInverseTransformer +{ + /// + /// Implements the eight-point inverse discrete cosine transform. + /// + private readonly struct DiscreteCosine8Operator : IHevcInverseTransformOperator + { + /// + public static int Size => 8; + + /// + public static bool UsesButterfly => true; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetCoefficient(int frequency, int position) + { + if (frequency == 0) + { + return 64; + } + + int angle = ((2 * position) + 1) * frequency * 4; + angle &= 127; + if (angle > 64) + { + angle = 128 - angle; + } + + // The second quadrant reuses the first-quadrant magnitude with a negative sign. + if (angle > 32) + { + return -DiscreteCosineMagnitudes[64 - angle]; + } + + return DiscreteCosineMagnitudes[angle]; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteSine4Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteSine4Operator.cs new file mode 100644 index 000000000..222914d4b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.DiscreteSine4Operator.cs @@ -0,0 +1,42 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the four-point inverse discrete sine transform operator. +/// +internal static partial class HevcInverseTransformer +{ + /// + /// Implements the four-point inverse discrete sine transform. + /// + private readonly struct DiscreteSine4Operator : IHevcInverseTransformOperator + { + /// + /// Gets the inverse-DST matrix in frequency-major order. + /// + private static ReadOnlySpan Coefficients => + [ + 29, 55, 74, 84, + 74, 74, 0, -74, + 84, -29, -74, 55, + 55, -84, 74, -29 + ]; + + /// + public static int Size => 4; + + /// + public static bool UsesButterfly => false; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetCoefficient(int frequency, int position) + { + return Coefficients[(frequency * Size) + position]; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs index 7ebc4c213..e300b6956 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs @@ -15,6 +15,15 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc; /// internal static partial class HevcInverseTransformer { + /// + /// Gets the common inverse-DCT magnitudes ordered on the pi-over-sixty-four angle grid. + /// + private static ReadOnlySpan DiscreteCosineMagnitudes => + [ + 90, 90, 90, 90, 89, 88, 87, 85, 83, 82, 80, 78, 75, 73, 70, 67, 64, + 61, 57, 54, 50, 46, 43, 38, 36, 31, 25, 22, 18, 13, 9, 4, 0 + ]; + /// /// Calculates the disjoint odd-frequency groups that seed the HEVC partial-butterfly reconstruction. /// @@ -23,7 +32,7 @@ internal static partial class HevcInverseTransformer /// The destination group rows. /// The number of independent lines transformed together. private static void PopulateButterflyGroups(ReadOnlySpan source, Span groups, int lineCount) - where TOperator : struct, IHevcInverseTransformOperator + where TOperator : struct, IHevcInverseTransformOperator { int size = TOperator.Size; int groupOffset = 0; @@ -76,7 +85,7 @@ internal static partial class HevcInverseTransformer int firstFrequency, int frequencyStep, int position) - where TOperator : struct, IHevcInverseTransformOperator + where TOperator : struct, IHevcInverseTransformOperator { ref int sourceBase = ref MemoryMarshal.GetReference(source); ref int destinationBase = ref MemoryMarshal.GetReference(destination); @@ -162,7 +171,7 @@ internal static partial class HevcInverseTransformer int shift, int minimum, int maximum) - where TOperator : struct, IHevcInverseTransformOperator + where TOperator : struct, IHevcInverseTransformOperator { int size = TOperator.Size; int combinedSize = 2; @@ -313,7 +322,7 @@ internal static partial class HevcInverseTransformer int shift, int minimum, int maximum) - where TOperator : struct, IHevcInverseTransformOperator + where TOperator : struct, IHevcInverseTransformOperator { ref int sourceBase = ref MemoryMarshal.GetReference(source); ref int destinationBase = ref MemoryMarshal.GetReference(destination); diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operator.cs new file mode 100644 index 000000000..8b4e814af --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operator.cs @@ -0,0 +1,34 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Defines the HEVC inverse-transform operator contract. +/// +internal static partial class HevcInverseTransformer +{ + /// + /// Defines one closed inverse-transform operation selected by the transform-unit syntax. + /// + private interface IHevcInverseTransformOperator + { + /// + /// Gets the transform side in samples. + /// + public static abstract int Size { get; } + + /// + /// Gets a value indicating whether the transform uses the partial-butterfly factorization. + /// + public static abstract bool UsesButterfly { get; } + + /// + /// Gets one inverse-transform matrix coefficient. + /// + /// The frequency-domain coordinate. + /// The spatial-domain coordinate. + /// The signed transform coefficient. + public static abstract int GetCoefficient(int frequency, int position); + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.OperatorImplementations.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.OperatorImplementations.cs deleted file mode 100644 index 4f1316232..000000000 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.OperatorImplementations.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; - -namespace SixLabors.ImageSharp.Formats.Heif.Hevc; - -internal static partial class HevcInverseTransformer -{ - /// - /// Gets the common HEVC inverse-DCT magnitudes ordered on the pi-over-sixty-four angle grid. - /// - private static ReadOnlySpan DiscreteCosineMagnitudes => - [ - 90, 90, 90, 90, 89, 88, 87, 85, 83, 82, 80, 78, 75, 73, 70, 67, 64, - 61, 57, 54, 50, 46, 43, 38, 36, 31, 25, 22, 18, 13, 9, 4, 0 - ]; - - /// - /// Gets the four-point HEVC inverse-DST matrix in frequency-major order. - /// - private static ReadOnlySpan DiscreteSine4Coefficients => - [ - 29, 55, 74, 84, - 74, 74, 0, -74, - 84, -29, -74, 55, - 55, -84, 74, -29 - ]; - - /// - /// Reconstructs one normative inverse-DCT coefficient from the common HEVC angle grid. - /// - /// The transform side. - /// The frequency-domain coordinate. - /// The spatial-domain coordinate. - /// The signed transform coefficient. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetDiscreteCosineCoefficient(int size, int frequency, int position) - { - if (frequency == 0) - { - return 64; - } - - int angle = ((2 * position) + 1) * frequency * (32 / size); - angle &= 127; - if (angle > 64) - { - angle = 128 - angle; - } - - // Angles in the second quadrant reuse the first-quadrant magnitude with a negative sign. Keeping the - // normative integer magnitudes in one compile-time span avoids runtime trigonometry and duplicate matrices. - if (angle > 32) - { - return -DiscreteCosineMagnitudes[64 - angle]; - } - - return DiscreteCosineMagnitudes[angle]; - } - - /// - /// Implements the four-point inverse discrete cosine transform. - /// - private readonly struct DiscreteCosine4Operator : IHevcInverseTransformOperator - { - /// - public static int Size => 4; - - /// - public static bool UsesButterfly => true; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetCoefficient(int frequency, int position) => GetDiscreteCosineCoefficient(Size, frequency, position); - } - - /// - /// Implements the four-point inverse discrete sine transform. - /// - private readonly struct DiscreteSine4Operator : IHevcInverseTransformOperator - { - /// - public static int Size => 4; - - /// - public static bool UsesButterfly => false; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetCoefficient(int frequency, int position) => DiscreteSine4Coefficients[(frequency * Size) + position]; - } - - /// - /// Implements the eight-point inverse discrete cosine transform. - /// - private readonly struct DiscreteCosine8Operator : IHevcInverseTransformOperator - { - /// - public static int Size => 8; - - /// - public static bool UsesButterfly => true; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetCoefficient(int frequency, int position) => GetDiscreteCosineCoefficient(Size, frequency, position); - } - - /// - /// Implements the sixteen-point inverse discrete cosine transform. - /// - private readonly struct DiscreteCosine16Operator : IHevcInverseTransformOperator - { - /// - public static int Size => 16; - - /// - public static bool UsesButterfly => true; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetCoefficient(int frequency, int position) => GetDiscreteCosineCoefficient(Size, frequency, position); - } - - /// - /// Implements the thirty-two-point inverse discrete cosine transform. - /// - private readonly struct DiscreteCosine32Operator : IHevcInverseTransformOperator - { - /// - public static int Size => 32; - - /// - public static bool UsesButterfly => true; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetCoefficient(int frequency, int position) => GetDiscreteCosineCoefficient(Size, frequency, position); - } -} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.cs index 4a350fcb1..2ab255f5f 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.cs @@ -16,32 +16,6 @@ internal static partial class HevcInverseTransformer /// private const int ResidualPrecision = 16; - /// - /// Defines one closed inverse-transform operation selected by the transform-unit syntax. - /// - /// The implementing operator type. - private interface IHevcInverseTransformOperator - where TOperator : struct, IHevcInverseTransformOperator - { - /// - /// Gets the transform side in samples. - /// - public static abstract int Size { get; } - - /// - /// Gets a value indicating whether the transform uses the partial-butterfly factorization. - /// - public static abstract bool UsesButterfly { get; } - - /// - /// Gets one inverse-transform matrix coefficient. - /// - /// The frequency-domain coordinate. - /// The spatial-domain coordinate. - /// The signed transform coefficient. - public static abstract int GetCoefficient(int frequency, int position); - } - /// /// Gets the scratch length required for the specified rectangular transform block. /// @@ -256,7 +230,7 @@ internal static partial class HevcInverseTransformer int shift, int minimum, int maximum) - where TOperator : struct, IHevcInverseTransformOperator + where TOperator : struct, IHevcInverseTransformOperator { if (!TOperator.UsesButterfly) { diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.LeftShiftTransformSkipOperator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.LeftShiftTransformSkipOperator.cs new file mode 100644 index 000000000..f714b5a6e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.LeftShiftTransformSkipOperator.cs @@ -0,0 +1,32 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +internal static partial class HevcResidualReconstructor +{ + /// + /// Applies the exact left shift used by high-bit-depth transform-skip reconstruction. + /// + private readonly struct LeftShiftTransformSkipOperator : ITransformSkipOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 values, int shift) => values << shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 values, int shift) => values << shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 values, int shift) => values << shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(int value, int shift) => value << shift; + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.Operator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.Operator.cs new file mode 100644 index 000000000..5604eba5b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.Operator.cs @@ -0,0 +1,47 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +internal static partial class HevcResidualReconstructor +{ + /// + /// Defines a closed transform-skip normalization operator for every SIMD width and the scalar tail. + /// + private interface ITransformSkipOperator + { + /// + /// Normalizes sixteen transform-skipped coefficients. + /// + /// The dequantized coefficients. + /// The nonnegative shift magnitude. + /// The reconstructed residuals. + static abstract Vector512 Invoke(Vector512 values, int shift); + + /// + /// Normalizes eight transform-skipped coefficients. + /// + /// The dequantized coefficients. + /// The nonnegative shift magnitude. + /// The reconstructed residuals. + static abstract Vector256 Invoke(Vector256 values, int shift); + + /// + /// Normalizes four transform-skipped coefficients. + /// + /// The dequantized coefficients. + /// The nonnegative shift magnitude. + /// The reconstructed residuals. + static abstract Vector128 Invoke(Vector128 values, int shift); + + /// + /// Normalizes one transform-skipped coefficient. + /// + /// The dequantized coefficient. + /// The nonnegative shift magnitude. + /// The reconstructed residual. + static abstract int Invoke(int value, int shift); + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.RightShiftTransformSkipOperator.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.RightShiftTransformSkipOperator.cs new file mode 100644 index 000000000..92d32e44e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.RightShiftTransformSkipOperator.cs @@ -0,0 +1,35 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +internal static partial class HevcResidualReconstructor +{ + /// + /// Applies the rounded right shift used by ordinary transform-skip reconstruction. + /// + private readonly struct RightShiftTransformSkipOperator : ITransformSkipOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 values, int shift) + => shift == 0 ? values : (values + Vector512.Create(1 << (shift - 1))) >> shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 values, int shift) + => shift == 0 ? values : (values + Vector256.Create(1 << (shift - 1))) >> shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 values, int shift) + => shift == 0 ? values : (values + Vector128.Create(1 << (shift - 1))) >> shift; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(int value, int shift) => shift == 0 ? value : (value + (1 << (shift - 1))) >> shift; + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs index c981c8209..1c7710540 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs @@ -16,7 +16,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc; /// and identity cases outside the row loops. Saturation to the residual range and clipping to sample depth occur at the /// same stage in every vector width and in the scalar tail. /// -internal static class HevcResidualReconstructor +internal static partial class HevcResidualReconstructor { /// /// The minimum residual sample represented by the decoder reconstruction pipeline. @@ -28,44 +28,6 @@ internal static class HevcResidualReconstructor /// private const int ResidualMaximum = short.MaxValue; - /// - /// Defines a closed transform-skip normalization operator for every SIMD width and the scalar tail. - /// - private interface ITransformSkipOperator - { - /// - /// Normalizes sixteen transform-skipped coefficients. - /// - /// The dequantized coefficients. - /// The nonnegative shift magnitude. - /// The reconstructed residuals. - static abstract Vector512 Invoke(Vector512 values, int shift); - - /// - /// Normalizes eight transform-skipped coefficients. - /// - /// The dequantized coefficients. - /// The nonnegative shift magnitude. - /// The reconstructed residuals. - static abstract Vector256 Invoke(Vector256 values, int shift); - - /// - /// Normalizes four transform-skipped coefficients. - /// - /// The dequantized coefficients. - /// The nonnegative shift magnitude. - /// The reconstructed residuals. - static abstract Vector128 Invoke(Vector128 values, int shift); - - /// - /// Normalizes one transform-skipped coefficient. - /// - /// The dequantized coefficient. - /// The nonnegative shift magnitude. - /// The reconstructed residual. - static abstract int Invoke(int value, int shift); - } - /// /// Copies one transquant-bypass coefficient block into residual sample order. /// @@ -613,51 +575,4 @@ internal static class HevcResidualReconstructor values += Vector128.Shuffle(values, Vector128.Create(4, 0, 1, 2)); return values + Vector128.Shuffle(values, Vector128.Create(4, 4, 0, 1)); } - - /// - /// Applies the rounded right shift used by ordinary transform-skip reconstruction. - /// - private readonly struct RightShiftTransformSkipOperator : ITransformSkipOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 values, int shift) - => shift == 0 ? values : (values + Vector512.Create(1 << (shift - 1))) >> shift; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 values, int shift) - => shift == 0 ? values : (values + Vector256.Create(1 << (shift - 1))) >> shift; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 values, int shift) - => shift == 0 ? values : (values + Vector128.Create(1 << (shift - 1))) >> shift; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(int value, int shift) => shift == 0 ? value : (value + (1 << (shift - 1))) >> shift; - } - - /// - /// Applies the exact left shift used by high-bit-depth transform-skip reconstruction. - /// - private readonly struct LeftShiftTransformSkipOperator : ITransformSkipOperator - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 values, int shift) => values << shift; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 values, int shift) => values << shift; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 values, int shift) => values << shift; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(int value, int shift) => value << shift; - } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index fc0ba961e..f20111190 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -40,18 +40,18 @@ public class Av1ForwardTransformTests [Fact] public void OneDimensionalOperatorsMatchAnalyticalReference() { - AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); } /// @@ -88,18 +88,18 @@ public class Av1ForwardTransformTests /// private static void AssertOneDimensionalOperators() { - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(32); - AssertOperator(64); - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(32); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); + AssertOperator(64); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); } /// @@ -108,32 +108,32 @@ public class Av1ForwardTransformTests /// The transform operator. /// The transform length. private static void AssertOperator(int length) - where TOperator : struct, IAv1ForwardTransform1dOperator + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { const int cosBit = 12; - AssertInt32Operator>(length, cosBit); + AssertInt32Vector128Operator(length, cosBit); if (Vector256.IsHardwareAccelerated) { - AssertInt32Operator>(length, cosBit); + AssertInt32Vector256Operator(length, cosBit); } if (Vector512.IsHardwareAccelerated) { - AssertInt32Operator>(length, cosBit); + AssertInt32Vector512Operator(length, cosBit); } - AssertInt16Operator>(length, cosBit); + AssertInt16Vector128Operator(length, cosBit); if (Avx2.IsSupported) { - AssertInt16Operator>(length, cosBit); + AssertInt16Vector256Operator(length, cosBit); } if (Avx512BW.IsSupported) { - AssertInt16Operator>(length, cosBit); + AssertInt16Vector512Operator(length, cosBit); } } @@ -144,7 +144,7 @@ public class Av1ForwardTransformTests /// The analytical transform definition. /// The transform length. private static void AssertOperatorAccuracy(Av1TransformType1d transformType, int length) - where TOperator : struct, IAv1ForwardTransform1dOperator + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { const int cosBit = 13; const int testBlockCount = 500; @@ -167,7 +167,7 @@ public class Av1ForwardTransformTests ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); - TOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, cosBit); + TOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, cosBit); Av1ReferenceTransform.ReferenceTransform1d(transformType, referenceInput, referenceOutput, length); // libaom permits seven integer coefficient units because each fixed-point butterfly rounds independently. @@ -184,24 +184,22 @@ public class Av1ForwardTransformTests } /// - /// Compares one Int32 vector representation with the scalar Int32 stage network lane by lane. + /// Compares the Vector128 Int32 representation with the scalar stage network lane by lane. /// /// The transform operator. - /// The SIMD value containing independent transform axes. /// The transform length. /// The fixed-point precision of the cosine constants. - private static void AssertInt32Operator(int length, int cosBit) - where TOperator : struct, IAv1ForwardTransform1dOperator - where TVector : struct + private static void AssertInt32Vector128Operator(int length, int cosBit) + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { - int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(int); - Av1TransformVector vectorValues = default; - Av1TransformVector vectorBuffer0 = default; - Av1TransformVector vectorBuffer1 = default; + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf>() / sizeof(int); + Av1TransformVector> vectorValues = default; + Av1TransformVector> vectorBuffer0 = default; + Av1TransformVector> vectorBuffer1 = default; for (int index = 0; index < length; index++) { - ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As, int>(ref vectorValues[index]); for (int lane = 0; lane < laneCount; lane++) { @@ -209,10 +207,10 @@ public class Av1ForwardTransformTests } } - ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref vectorValues); - nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf(); + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As>, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf>(); - TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); for (int lane = 0; lane < laneCount; lane++) { @@ -227,35 +225,241 @@ public class Av1ForwardTransformTests ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); - TOperator.Transform(ref scalarValuesBase, sizeof(int), sizeof(int), ref scalarBuffer0, ref scalarBuffer1, cosBit); + TOperator.Transform(ref scalarValuesBase, sizeof(int), sizeof(int), ref scalarBuffer0, ref scalarBuffer1, cosBit); for (int index = 0; index < length; index++) { - ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As, int>(ref vectorValues[index]); Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } /// - /// Compares one Int16 vector representation with the scalar Int16 stage network lane by lane. + /// Compares the Vector256 Int32 representation with the scalar stage network lane by lane. /// /// The transform operator. - /// The SIMD value containing independent transform axes. /// The transform length. /// The fixed-point precision of the cosine constants. - private static void AssertInt16Operator(int length, int cosBit) - where TOperator : struct, IAv1ForwardTransform1dOperator - where TVector : struct + private static void AssertInt32Vector256Operator(int length, int cosBit) + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { - int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(short); - Av1TransformVector vectorValues = default; - Av1TransformVector vectorBuffer0 = default; - Av1TransformVector vectorBuffer1 = default; + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf>() / sizeof(int); + Av1TransformVector> vectorValues = default; + Av1TransformVector> vectorBuffer0 = default; + Av1TransformVector> vectorBuffer1 = default; for (int index = 0; index < length; index++) { - ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As, int>(ref vectorValues[index]); + + for (int lane = 0; lane < laneCount; lane++) + { + System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetInputValue(index, lane); + } + } + + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As>, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf>(); + + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); + + for (int lane = 0; lane < laneCount; lane++) + { + Av1TransformVector scalarValues = default; + Av1TransformVector scalarBuffer0 = default; + Av1TransformVector scalarBuffer1 = default; + + for (int index = 0; index < length; index++) + { + scalarValues[index] = GetInputValue(index, lane); + } + + ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); + + TOperator.Transform(ref scalarValuesBase, sizeof(int), sizeof(int), ref scalarBuffer0, ref scalarBuffer1, cosBit); + + for (int index = 0; index < length; index++) + { + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As, int>(ref vectorValues[index]); + Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); + } + } + } + + /// + /// Compares the Vector512 Int32 representation with the scalar stage network lane by lane. + /// + /// The transform operator. + /// The transform length. + /// The fixed-point precision of the cosine constants. + private static void AssertInt32Vector512Operator(int length, int cosBit) + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + { + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf>() / sizeof(int); + Av1TransformVector> vectorValues = default; + Av1TransformVector> vectorBuffer0 = default; + Av1TransformVector> vectorBuffer1 = default; + + for (int index = 0; index < length; index++) + { + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As, int>(ref vectorValues[index]); + + for (int lane = 0; lane < laneCount; lane++) + { + System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetInputValue(index, lane); + } + } + + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As>, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf>(); + + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); + + for (int lane = 0; lane < laneCount; lane++) + { + Av1TransformVector scalarValues = default; + Av1TransformVector scalarBuffer0 = default; + Av1TransformVector scalarBuffer1 = default; + + for (int index = 0; index < length; index++) + { + scalarValues[index] = GetInputValue(index, lane); + } + + ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); + + TOperator.Transform(ref scalarValuesBase, sizeof(int), sizeof(int), ref scalarBuffer0, ref scalarBuffer1, cosBit); + + for (int index = 0; index < length; index++) + { + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As, int>(ref vectorValues[index]); + Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); + } + } + } + + /// + /// Compares the Vector128 Int16 representation with the scalar stage network lane by lane. + /// + /// The transform operator. + /// The transform length. + /// The fixed-point precision of the cosine constants. + private static void AssertInt16Vector128Operator(int length, int cosBit) + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + { + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf>() / sizeof(short); + Av1TransformVector> vectorValues = default; + Av1TransformVector> vectorBuffer0 = default; + Av1TransformVector> vectorBuffer1 = default; + + for (int index = 0; index < length; index++) + { + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As, short>(ref vectorValues[index]); + + for (int lane = 0; lane < laneCount; lane++) + { + System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetPackedInputValue(index, lane); + } + } + + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As>, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf>(); + + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); + + for (int lane = 0; lane < laneCount; lane++) + { + Av1TransformVector scalarValues = default; + Av1TransformVector scalarBuffer0 = default; + Av1TransformVector scalarBuffer1 = default; + + for (int index = 0; index < length; index++) + { + scalarValues[index] = GetPackedInputValue(index, lane); + } + + ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); + + TOperator.Transform(ref scalarValuesBase, sizeof(short), sizeof(short), ref scalarBuffer0, ref scalarBuffer1, cosBit); + + for (int index = 0; index < length; index++) + { + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As, short>(ref vectorValues[index]); + Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); + } + } + } + + /// + /// Compares the Vector256 Int16 representation with the scalar stage network lane by lane. + /// + /// The transform operator. + /// The transform length. + /// The fixed-point precision of the cosine constants. + private static void AssertInt16Vector256Operator(int length, int cosBit) + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + { + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf>() / sizeof(short); + Av1TransformVector> vectorValues = default; + Av1TransformVector> vectorBuffer0 = default; + Av1TransformVector> vectorBuffer1 = default; + + for (int index = 0; index < length; index++) + { + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As, short>(ref vectorValues[index]); + + for (int lane = 0; lane < laneCount; lane++) + { + System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetPackedInputValue(index, lane); + } + } + + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As>, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf>(); + + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); + + for (int lane = 0; lane < laneCount; lane++) + { + Av1TransformVector scalarValues = default; + Av1TransformVector scalarBuffer0 = default; + Av1TransformVector scalarBuffer1 = default; + + for (int index = 0; index < length; index++) + { + scalarValues[index] = GetPackedInputValue(index, lane); + } + + ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); + + TOperator.Transform(ref scalarValuesBase, sizeof(short), sizeof(short), ref scalarBuffer0, ref scalarBuffer1, cosBit); + + for (int index = 0; index < length; index++) + { + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As, short>(ref vectorValues[index]); + Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); + } + } + } + + /// + /// Compares the Vector512 Int16 representation with the scalar stage network lane by lane. + /// + /// The transform operator. + /// The transform length. + /// The fixed-point precision of the cosine constants. + private static void AssertInt16Vector512Operator(int length, int cosBit) + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + { + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf>() / sizeof(short); + Av1TransformVector> vectorValues = default; + Av1TransformVector> vectorBuffer0 = default; + Av1TransformVector> vectorBuffer1 = default; + + for (int index = 0; index < length; index++) + { + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As, short>(ref vectorValues[index]); for (int lane = 0; lane < laneCount; lane++) { @@ -263,10 +467,10 @@ public class Av1ForwardTransformTests } } - ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref vectorValues); - nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf(); + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As>, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf>(); - TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); for (int lane = 0; lane < laneCount; lane++) { @@ -281,11 +485,11 @@ public class Av1ForwardTransformTests ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); - TOperator.Transform(ref scalarValuesBase, sizeof(short), sizeof(short), ref scalarBuffer0, ref scalarBuffer1, cosBit); + TOperator.Transform(ref scalarValuesBase, sizeof(short), sizeof(short), ref scalarBuffer0, ref scalarBuffer1, cosBit); for (int index = 0; index < length; index++) { - ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As, short>(ref vectorValues[index]); Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } @@ -368,40 +572,40 @@ public class Av1ForwardTransformTests switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; } } @@ -415,45 +619,45 @@ public class Av1ForwardTransformTests /// The destination reference coefficients. /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceRow(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TColumnOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; } } @@ -472,8 +676,8 @@ public class Av1ForwardTransformTests int stride, Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1ForwardTransform1dOperator - where TRowOperator : struct, IAv1ForwardTransform1dOperator + where TColumnOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TRowOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); @@ -494,7 +698,7 @@ public class Av1ForwardTransformTests ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); - TColumnOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitColumn); + TColumnOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitColumn); int destinationColumn = config.FlipLeftToRight ? width - column - 1 : column; for (int row = 0; row < height; row++) @@ -514,7 +718,7 @@ public class Av1ForwardTransformTests ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); - TRowOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitRow); + TRowOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitRow); for (int column = 0; column < outputWidth; column++) { diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 15b0d98f9..753454afc 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -5,7 +5,6 @@ using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -55,11 +54,11 @@ public class Av1InverseTransformTests /// private static void AssertDctOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); - AssertOperatorParity(64); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); + AssertOperatorParity(32); + AssertOperatorParity(64); } /// @@ -67,9 +66,9 @@ public class Av1InverseTransformTests /// private static void AssertAdstOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); } /// @@ -77,10 +76,10 @@ public class Av1InverseTransformTests /// private static void AssertIdentityOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); + AssertOperatorParity(32); } /// @@ -110,14 +109,14 @@ public class Av1InverseTransformTests Av1TransformVector> adstOutput256 = default; Av1TransformVector> adstStep256 = default; - Av1Adst4Inverse1dOperator.Transform( + Av1Inverse2dTransformer.Adst4Operator.Transform( ref adstInput128, ref adstOutput128, ref adstStep128, cosBit, stageRange); - Av1Adst4Inverse1dOperator.Transform( + Av1Inverse2dTransformer.Adst4Operator.Transform( ref adstInput256, ref adstOutput256, ref adstStep256, @@ -150,7 +149,7 @@ public class Av1InverseTransformTests 262_143, -262_144); - AssertWidenedIdentityOperator( + AssertWidenedIdentityOperator( 4, identityInput128, Vector128.Create(741_503, -741_504, 741_501, -741_503), @@ -158,7 +157,7 @@ public class Av1InverseTransformTests Vector256.Create(741_503, -741_504, 741_501, -741_503, 524_322, -524_323, 370_751, -370_752), stageRange); - AssertWidenedIdentityOperator( + AssertWidenedIdentityOperator( 16, identityInput128, Vector128.Create(1_483_005, -1_483_008, 1_483_002, -1_483_005), @@ -184,7 +183,7 @@ public class Av1InverseTransformTests Vector256 input256, Vector256 expected256, Av1TransformStageRange stageRange) - where TOperator : struct, IAv1Transform1dOperator + where TOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { const int cosBit = 12; Av1TransformVector> values128 = default; @@ -270,18 +269,18 @@ public class Av1InverseTransformTests [Fact] public void ForwardAndInverseOperatorPairsReconstructTheirInput() { - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size8x8, 2, 2); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size16x16, 3, 3); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size32x32, 4, 4); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size64x64, 5, 5); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size8x8, 2, 2); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size16x16, 3, 3); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size8x8, 2, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size16x16, 3, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size8x8, 2, 2); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size16x16, 3, 3); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size32x32, 4, 4); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size64x64, 5, 5); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size8x8, 2, 2); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size16x16, 3, 3); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size8x8, 2, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size16x16, 3, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); } /// @@ -598,7 +597,7 @@ public class Av1InverseTransformTests /// The inverse transform operator. /// The transform length. private static void AssertOperatorParity(int length) - where TOperator : struct, IAv1Transform1dOperator + where TOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { const int cosBit = 12; Av1TransformStageRange stageRange = default; @@ -672,8 +671,8 @@ public class Av1InverseTransformTests /// The power-of-two scale applied by the operator pair. /// The maximum permitted reconstruction error. private static void AssertRoundTrip(Av1TransformType transformType, Av1TransformSize transformSize, int scaleLog2, int allowedError) - where TForwardOperator : struct, IAv1ForwardTransform1dOperator - where TInverseOperator : struct, IAv1Transform1dOperator + where TForwardOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TInverseOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { const int bitDepth = 10; const int testBlockCount = 30; @@ -699,7 +698,7 @@ public class Av1InverseTransformTests ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); - TForwardOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, forwardConfig.CosBitColumn); + TForwardOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, forwardConfig.CosBitColumn); for (int index = 0; index < length; index++) { @@ -732,40 +731,40 @@ public class Av1InverseTransformTests switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; default: Assert.Fail($"Unexpected column function {config.TransformFunctionTypeColumn} for {transformType} {transformSize}."); @@ -786,45 +785,45 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct32: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct64: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity32: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; default: Assert.Fail($"Unexpected row function {config.TransformFunctionTypeRow} for {transformType} {transformSize}."); @@ -846,8 +845,8 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator + where TRowOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -897,8 +896,8 @@ public class Av1InverseTransformTests int[] coefficients, Av1TransformSize transformSize, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator + where TRowOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { const int bitDepth = 8; int width = transformSize.GetWidth(); @@ -923,10 +922,10 @@ public class Av1InverseTransformTests Array.Fill(scalar, byte.MaxValue); Array.Fill(vector128, byte.MaxValue); - Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dScalar( coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); - Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector128( coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); @@ -937,7 +936,7 @@ public class Av1InverseTransformTests int[] vector256Workspace = new int[workspaceLength]; Array.Fill(vector256, byte.MaxValue); - Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector256( coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); Assert.Equal(scalar, vector256); @@ -958,8 +957,8 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator + where TRowOperator : struct, Av1Inverse2dTransformer.IAv1Transform1dOperator { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -984,10 +983,10 @@ public class Av1InverseTransformTests Array.Fill(scalar, short.MinValue); Array.Fill(vector128, short.MinValue); - Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dScalar( coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); - Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector128( coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); @@ -998,7 +997,7 @@ public class Av1InverseTransformTests int[] vector256Workspace = new int[workspaceLength]; Array.Fill(vector256, short.MinValue); - Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector256( coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); Assert.Equal(scalar, vector256);