// Copyright (c) Six Labors. // Licensed under the Six Labors Split License. using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; /// /// Verifies AV1 forward transform arithmetic, dispatch, layout, and allocation behavior. /// [Trait("Format", "Avif")] public class Av1ForwardTransformTests { /// /// The hardware configurations covering every transform vector tier and the scalar fallback. /// private const HwIntrinsics TransformConfigurations = HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; /// /// Gets every normative transform size, type, and bit-depth combination shared with the inverse suite. /// public static TheoryData ValidTransformCases { get; } = CreateValidTransformCases(); /// /// Verifies every one-dimensional stage network across its scalar and available vector representations. /// [Fact] public void OneDimensionalOperatorsMatchAcrossHardwareWidths() => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertOneDimensionalOperators, TransformConfigurations); /// /// Verifies every one-dimensional stage network against the independent analytical transform definition. /// [Fact] public void OneDimensionalOperatorsMatchAnalyticalReference() { AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); } /// /// Verifies every permitted size, type, and bit-depth combination against the direct scalar two-axis definition. /// [Fact] public void TwoDimensionalPipelineMatchesScalarReferenceAcrossHardwareConfigurations() => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertTwoDimensionalPipeline, TransformConfigurations); /// /// Verifies that the complete transform dispatcher reuses caller-owned workspace. /// [Fact] public void TransformDispatchDoesNotAllocatePerBlock() { const int width = 8; short[] input = new short[width * width]; int[] output = new int[input.Length]; int[] workspace = new int[Av1TransformWorkspace.MaximumLength]; Av1ForwardTransformer.Transform2d(input, output, width, Av1TransformType.DctDct, Av1TransformSize.Size8x8, 8, workspace); long before = GC.GetAllocatedBytesForCurrentThread(); for (int iteration = 0; iteration < 32; iteration++) { Av1ForwardTransformer.Transform2d(input, output, width, Av1TransformType.DctDct, Av1TransformSize.Size8x8, 8, workspace); } Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); } /// /// Exercises every DCT, ADST, and identity stage network using both Int16 and Int32 lane arithmetic. /// private static void AssertOneDimensionalOperators() { AssertOperator(4); AssertOperator(8); AssertOperator(16); AssertOperator(32); AssertOperator(64); AssertOperator(4); AssertOperator(8); AssertOperator(16); AssertOperator(4); AssertOperator(8); AssertOperator(16); AssertOperator(32); } /// /// Compares one stage network across all available scalar and vector representations. /// /// The transform operator. /// The transform length. private static void AssertOperator(int length) where TOperator : struct, IAv1ForwardTransform1dOperator { const int cosBit = 12; AssertInt32Operator>(length, cosBit); if (Vector256.IsHardwareAccelerated) { AssertInt32Operator>(length, cosBit); } if (Vector512.IsHardwareAccelerated) { AssertInt32Operator>(length, cosBit); } AssertInt16Operator>(length, cosBit); if (Avx2.IsSupported) { AssertInt16Operator>(length, cosBit); } if (Avx512BW.IsSupported) { AssertInt16Operator>(length, cosBit); } } /// /// Compares one integer stage network with the analytical transform used by the libaom forward-transform tests. /// /// The transform operator. /// The analytical transform definition. /// The transform length. private static void AssertOperatorAccuracy(Av1TransformType1d transformType, int length) where TOperator : struct, IAv1ForwardTransform1dOperator { const int cosBit = 13; const int testBlockCount = 500; const int maximumCoefficientError = 7; Random random = new(0); double[] referenceInput = new double[length]; double[] referenceOutput = new double[length]; Av1TransformVector values = default; Av1TransformVector buffer0 = default; Av1TransformVector buffer1 = default; for (int block = 0; block < testBlockCount; block++) { for (int index = 0; index < length; index++) { int input = random.Next(1024) - random.Next(1024); values[index] = input; referenceInput[index] = input; } ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); TOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, cosBit); Av1ReferenceTransform.ReferenceTransform1d(transformType, referenceInput, referenceOutput, length); // libaom permits seven integer coefficient units because each fixed-point butterfly rounds independently. for (int index = 0; index < length; index++) { int expected = (int)Math.Round(referenceOutput[index], MidpointRounding.AwayFromZero); int error = Math.Abs(values[index] - expected); Assert.True( error <= maximumCoefficientError, $"{typeof(TOperator).Name} coefficient {index}: expected {expected}, actual {values[index]}, error {error}."); } } } /// /// Compares one Int32 vector representation with the scalar Int32 stage network lane by lane. /// /// The transform operator. /// The SIMD value containing independent transform axes. /// The transform length. /// The fixed-point precision of the cosine constants. private static void AssertInt32Operator(int length, int cosBit) where TOperator : struct, IAv1ForwardTransform1dOperator where TVector : struct { int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(int); Av1TransformVector vectorValues = default; Av1TransformVector vectorBuffer0 = default; Av1TransformVector vectorBuffer1 = default; for (int index = 0; index < length; index++) { ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); for (int lane = 0; lane < laneCount; lane++) { System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetInputValue(index, lane); } } ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref vectorValues); nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf(); TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); for (int lane = 0; lane < laneCount; lane++) { Av1TransformVector scalarValues = default; Av1TransformVector scalarBuffer0 = default; Av1TransformVector scalarBuffer1 = default; for (int index = 0; index < length; index++) { scalarValues[index] = GetInputValue(index, lane); } ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); TOperator.Transform(ref scalarValuesBase, sizeof(int), sizeof(int), ref scalarBuffer0, ref scalarBuffer1, cosBit); for (int index = 0; index < length; index++) { ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } /// /// Compares one Int16 vector representation with the scalar Int16 stage network lane by lane. /// /// The transform operator. /// The SIMD value containing independent transform axes. /// The transform length. /// The fixed-point precision of the cosine constants. private static void AssertInt16Operator(int length, int cosBit) where TOperator : struct, IAv1ForwardTransform1dOperator where TVector : struct { int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(short); Av1TransformVector vectorValues = default; Av1TransformVector vectorBuffer0 = default; Av1TransformVector vectorBuffer1 = default; for (int index = 0; index < length; index++) { ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); for (int lane = 0; lane < laneCount; lane++) { System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetPackedInputValue(index, lane); } } ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref vectorValues); nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf(); TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); for (int lane = 0; lane < laneCount; lane++) { Av1TransformVector scalarValues = default; Av1TransformVector scalarBuffer0 = default; Av1TransformVector scalarBuffer1 = default; for (int index = 0; index < length; index++) { scalarValues[index] = GetPackedInputValue(index, lane); } ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); TOperator.Transform(ref scalarValuesBase, sizeof(short), sizeof(short), ref scalarBuffer0, ref scalarBuffer1, cosBit); for (int index = 0; index < length; index++) { ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } /// /// Exercises the complete normative transform matrix for the active hardware configuration. /// private static void AssertTwoDimensionalPipeline() { for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) { for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) { Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, 8); if (!config.IsAllowed()) { continue; } for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) { AssertTwoDimensionalCase(transformType, transformSize, bitDepth); } } } } /// /// Compares one complete transform with the direct scalar two-axis definition. /// /// The compound transform type. /// The transform-block dimensions. /// The source sample bit depth. private static void AssertTwoDimensionalCase(Av1TransformType transformType, Av1TransformSize transformSize, int bitDepth) { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); int inputStride = width + 3; Av1TransformSize adjustedSize = transformSize.GetAdjusted(); int coefficientCount = adjustedSize.GetWidth() * adjustedSize.GetHeight(); short[] input = new short[inputStride * height]; int sampleMaximum = (1 << bitDepth) - 1; for (int row = 0; row < height; row++) { for (int column = 0; column < width; column++) { int index = (row * width) + column; input[(row * inputStride) + column] = (short)((index & 3) switch { 0 => sampleMaximum, 1 => -sampleMaximum, 2 => ((index * 73) % ((2 * sampleMaximum) + 1)) - sampleMaximum, _ => 0, }); } } int[] expected = new int[coefficientCount]; int[] actual = new int[coefficientCount]; int[] workspace = new int[Av1TransformWorkspace.MaximumLength]; Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); DispatchReferenceColumn(input, inputStride, expected, ref config); Av1ForwardTransformer.Transform2d(input, actual, (uint)inputStride, transformType, transformSize, bitDepth, workspace); Assert.Equal(expected, actual); } /// /// Selects the scalar reference column operator. /// /// The spatial residual samples. /// The number of input samples between rows. /// The destination reference coefficients. /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceColumn(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) { switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: DispatchReferenceRow(input, stride, output, ref config); break; } } /// /// Selects the scalar reference row operator. /// /// The column transform operator. /// The spatial residual samples. /// The number of input samples between rows. /// The destination reference coefficients. /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceRow(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) where TColumnOperator : struct, IAv1ForwardTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: TransformReference(input, stride, output, ref config); break; } } /// /// Applies the direct scalar column and row transform definition used as the layout and dispatch oracle. /// /// The column transform operator. /// The row transform operator. /// The spatial residual samples. /// The number of input samples between rows. /// The destination reference coefficients. /// The resolved transform functions, shifts, and axis orientation. private static void TransformReference( Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) where TColumnOperator : struct, IAv1ForwardTransform1dOperator where TRowOperator : struct, IAv1ForwardTransform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); int outputWidth = Math.Min(width, 32); int outputHeight = Math.Min(height, 32); int[] intermediate = new int[width * height]; Av1TransformVector values = default; Av1TransformVector buffer0 = default; Av1TransformVector buffer1 = default; for (int column = 0; column < width; column++) { for (int row = 0; row < height; row++) { int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; values[row] = input[(sourceRow * stride) + column] << config.Shift0; } ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); TColumnOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitColumn); int destinationColumn = config.FlipLeftToRight ? width - column - 1 : column; for (int row = 0; row < height; row++) { intermediate[(row * width) + destinationColumn] = Av1Math.RoundShift(values[row], -config.Shift1); } } bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; for (int row = 0; row < outputHeight; row++) { for (int column = 0; column < width; column++) { values[column] = intermediate[(row * width) + column]; } ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); TRowOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitRow); for (int column = 0; column < outputWidth; column++) { int value = Av1Math.RoundShift(values[column], -config.Shift2); output[(row * outputWidth) + column] = normalizeRectangle ? Av1Transform1dMath.HalfButterfly(Av1Transform1dMath.NewSqrt2, value, 0, 0, Av1Transform1dMath.NewSqrt2Bits) : value; } } } /// /// Gets a deterministic signed thirty-two-bit transform input. /// /// The transform position. /// The independent SIMD lane. /// The deterministic input value. private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 8191) - 4095; /// /// Gets a deterministic signed sixteen-bit input including overflow-sensitive edge values. /// /// The transform position. /// The independent SIMD lane. /// The deterministic packed input value. private static short GetPackedInputValue(int index, int lane) => (short)((index + lane) % 5 switch { 0 => short.MaxValue, 1 => short.MinValue, 2 => 255, 3 => -255, _ => ((index * 73) + (lane * 151)) % 511 - 255, }); /// /// Creates the complete normative transform matrix shared by the forward and inverse tests. /// /// Every permitted transform type, transform size, and AV1 image bit depth. private static TheoryData CreateValidTransformCases() { TheoryData cases = []; for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) { for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) { Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, 8); if (!config.IsAllowed()) { continue; } for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) { cases.Add((int)transformType, (int)transformSize, bitDepth); } } } return cases; } }