diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index c4d8653540..acbce5ece9 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -29,18 +29,19 @@ Checkboxes may be marked complete only when the implementation and the verificat ## Active and queued work -- [ ] **Active:** eliminate per-call scratch allocation across every AV1 forward and inverse 1-D transform, replace shared mutable transform scratch with operation-owned workspace, and implement libaom-shaped bulk transform paths with scalar parity. +- [x] **Completed:** eliminate per-call scratch allocation across every AV1 forward and inverse 1-D transform, replace shared mutable transform scratch with operation-owned workspace, and implement libaom-shaped bulk transform paths with scalar parity. - [x] Inventory the existing hot-path scratch use: 15 forward/inverse transformer types contain 30 `stackalloc int[...]` sites. - [x] Refactor every affected forward transform to reuse caller-owned workspace and its output buffer for staged ping-pong storage. - [x] Refactor every affected inverse transform to reuse caller-owned workspace and its output buffer for staged ping-pong storage. - [x] Remove the shared mutable `Av1ForwardTransformer.TemporaryCoefficientsBuffer` and give each concurrent transform operation exclusive workspace ownership. - [x] Replace the instance-transformer interface with stateless forward and inverse operator structs following the JPEG color-transform static-generic operator pattern. - [x] Select transform type, size, bit depth, and ISA once at the 2-D block boundary rather than dispatching through an interface for every row and column. - - [ ] Port the applicable libaom bulk forward-transform kernels using the existing ImageSharp `Vector128` and `Vector256` infrastructure while retaining the scalar oracle; add a managed `Vector512` path only if complete-block measurements justify extending beyond the pinned upstream SIMD implementations. - - [ ] Port the applicable libaom bulk inverse-transform kernels using the same tables, rounding, saturation, and clipping rules as the scalar oracle. + - [x] Port the applicable libaom bulk forward-transform kernels using the existing ImageSharp `Vector128` and `Vector256` infrastructure while retaining the scalar oracle; add a managed `Vector512` path only if complete-block measurements justify extending beyond the pinned upstream SIMD implementations. + - [x] Port the applicable libaom bulk inverse-transform kernels using the same tables, rounding, saturation, and clipping rules as the scalar oracle. - [x] Document scratch ownership, stage-buffer alternation, fixed-point rounding, lane layout, transposition, and scalar fallback decisions at their implementation points. - [x] Add complete-block BenchmarkDotNet coverage for scalar, `Vector128`, `Vector256`, and runtime-dispatched 8x8 and 32x32 forward/inverse DCT paths, including managed-allocation reporting. - - [ ] Verify every scalar and hardware path across all transform types, sizes, bit depths, edge blocks, and overflow cases, then record representative end-to-end timings and allocations. + - [x] Verify every scalar and hardware path across all transform types, sizes, bit depths, edge blocks, and bounded overflow cases, then record representative complete-block timings and allocations. + - Verification covers all 159 normative size/type combinations at 8, 10, and 12 bits with padded input, prediction, and destination strides. Each of the 477 configurations compares scalar and `Vector128` output, and every configuration that contains a complete eight-lane tile also compares `Vector256` output. The 989-test focused suite passes with normal AVX2 dispatch, with AVX2 disabled, and with all hardware intrinsics disabled. - [ ] **Queued:** restore bounded animated HEIC and AVIF image-sequence scope, including the required image-level and per-frame metadata contracts, without introducing unrelated ISO BMFF surfaces. - [ ] Reconcile the still-image-only statements in this plan with the required animated HEIC and AVIF completion scope before sequence implementation begins. - [ ] Define the ImageSharp image-level sequence metadata and per-frame metadata contracts, including observable timing, repetition, color, alpha, orientation, and profile behavior. @@ -213,7 +214,7 @@ This assessment is based on the current source after the upstream ImageSharp mer - Loop-restoration unit parsing records tile-local switchable/Wiener/self-guided filter selections and coefficients in frame-owned plane grids, including super-resolution-adjusted unit corners and the corrected conditional 64x64-superblock unit-size bit. The active restoration stage implements the normative unit geometry, striped deblocked boundaries, Wiener filtering, self-guided projection, and 8/10/12-bit clipping, while reusing the existing SIMD-backed super-resolution and adjacent multiply/add primitives. Independently encoded fixtures covering every parameter set, plane layout, bit depth, and frame-edge geometry are still required. Other normative independently decodable still-image syntax paths still contain `NotImplementedException` or equivalent unsupported branches. Tile-local palette CDF adaptation is present; the remaining still-image frame-context behavior requires a separate source audit without introducing sequence playback state. - The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active intra-prediction, inverse-transform, and block-reconstruction path selects native 16-bit samples for 10/12-bit frames and has focused pipeline wiring coverage. Chroma-from-luma storage, subsampling, parameter derivation, U/V sharing, and 8/10/12-bit prediction are active; independently encoded high-bit-depth and chroma-from-luma AVIF conformance files are still required. - `Av1YuvConverter` now consumes the signaled full or limited range, every non-reserved AV1 H.273 matrix coefficient, transfer characteristics where the matrix definition requires them, subsampling, and chroma sample position for 8, 10, and 12-bit output. Its high-bit-depth decode and encode paths use allocator-backed `Rgb48` rows and the existing `PixelOperations` conversions, avoiding the former eight-bit intermediate. Encoder conversion covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with libavif-compatible box averaging. Identity, full/limited-range YCgCo, the fixed non-constant-luminance matrices, both fixed and chromaticity-derived constant/non-constant-luminance systems, SMPTE ST 2085, and PQ/HLG ICtCp are active in both directions. Independent vectors for every matrix, transfer, range, bit depth, sampling layout, and chroma position remain required before the complete color matrix is externally verified. -- Forward and inverse transforms now use operation-owned allocator workspace and stateless static-generic operator structs for every valid DCT, ADST, and identity size. Named configuration factories keep the encoder's three shifts and variable cosine precision separate from the decoder's two shifts, fixed 12-bit cosine precision, and 8/10/12-bit clamp ranges. The two-dimensional traversal selects `Vector256`, then `Vector128`, with scalar only when hardware vectorization is unavailable. Focused tests prove exact scalar/SIMD parity, the complete inverse shift/range tables, and zero per-block managed allocations. On the measured .NET 10 AVX2 host, runtime-dispatched 8x8 DCT blocks take 102.42 ns forward and 112.14 ns inverse versus 441.19 ns and 602.52 ns scalar; 32x32 blocks take 1.439 microseconds forward and 8.825 microseconds inverse versus 9.112 microseconds and 23.455 microseconds scalar. BenchmarkDotNet reports no managed allocation for any measured path. The complete bit-depth, edge, overflow, upstream-kernel, and independent-host matrix remains open. +- Forward and inverse transforms now use operation-owned allocator workspace and stateless static-generic operator structs for every valid DCT, ADST, and identity size. Named configuration factories keep the encoder's three shifts and variable cosine precision separate from the decoder's two shifts, fixed 12-bit cosine precision, and 8/10/12-bit clamp ranges. The two-dimensional traversal selects `Vector256`, then `Vector128`, with scalar only when hardware vectorization is unavailable. Focused tests prove exact scalar/SIMD parity for all 159 normative size/type combinations at every supported bit depth, padded edge strides, bounded residual extrema, the complete inverse shift/range tables, and zero per-block managed allocations. The 989-test suite passes with normal AVX2 dispatch, AVX2 disabled, and all hardware intrinsics disabled. On the measured .NET 10 AVX2 host, runtime-dispatched 8x8 DCT blocks take 102.42 ns forward and 112.14 ns inverse versus 441.19 ns and 602.52 ns scalar; 32x32 blocks take 1.439 microseconds forward and 8.825 microseconds inverse versus 9.112 microseconds and 23.455 microseconds scalar. BenchmarkDotNet reports no managed allocation for any measured path. A managed `Vector512` path remains deliberately absent: pinned libaom has an optional Highway AVX-512 low-bit-depth forward path, but no matching complete inverse and high-bit-depth transform family, and the complete-block results do not justify maintaining a separate partial algorithm. - The production prediction and nonlinear self-guided paths remain predominantly scalar. Transform traversal is SIMD-first on supported hardware, while normative super-resolution and Wiener horizontal products reuse ImageSharp's cross-platform adjacent multiply/add SIMD helper with exact scalar fallbacks; further SIMD work must preserve these scalar reconstruction oracles. ### AV1 encoder @@ -446,7 +447,7 @@ SIMD work begins after the corresponding scalar stage has independent correctnes Tasks: - [ ] Remove known avoidable allocations first: per-transform arrays, the intermediate RGB image, repeated block scratch arrays, and file-sized buffering. - - [ ] Complete the active AV1 forward and inverse transform-family checklist above. + - [x] Complete the active AV1 forward and inverse transform-family checklist above. - [ ] Benchmark codec-specific costs for CABAC/range decode, inverse transforms, still-image prediction, deblocking, SAO, CDEF, restoration, chroma upsampling, color conversion, alpha packing, and grid copies. - [ ] Implement vector paths only for confirmed hot loops, using existing `Vector128`, `Vector256`, and `Vector512` helper and dispatch patterns where supported. - [ ] Prioritize shared color conversion and pixel packing, chroma upsampling, inverse-transform add-and-clip, intra predictors, HEVC deblock/SAO, AV1 loop filter/CDEF/restoration, and contiguous grid copies. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dFlipConfiguration.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dFlipConfiguration.cs index 01ceedb810..3246bca19e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dFlipConfiguration.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dFlipConfiguration.cs @@ -369,15 +369,19 @@ internal ref struct Av1Transform2dFlipConfiguration /// /// when the transform combination is valid for the transform size. public bool IsAllowed() - => this.TransformSize switch + { + // AV1 selects the legal transform set from the block's square-up size: blocks up to 16x16 allow all sixteen + // types, a 32x32 square-up allows DCT and identity, and larger square-up sizes allow DCT only. + return this.TransformSize switch { - Av1TransformSize.Size32x32 => - this.TransformType is Av1TransformType.DctDct or Av1TransformType.Identity or Av1TransformType.VerticalDct or Av1TransformType.HorizontalDct, - Av1TransformSize.Size32x64 or Av1TransformSize.Size64x32 or Av1TransformSize.Size16x64 or Av1TransformSize.Size64x16 => this.TransformType == Av1TransformType.DctDct, - Av1TransformSize.Size16x32 or Av1TransformSize.Size32x16 or Av1TransformSize.Size64x64 or Av1TransformSize.Size8x32 or Av1TransformSize.Size32x8 => + Av1TransformSize.Size32x32 or Av1TransformSize.Size16x32 or Av1TransformSize.Size32x16 or + Av1TransformSize.Size8x32 or Av1TransformSize.Size32x8 => this.TransformType is Av1TransformType.DctDct or Av1TransformType.Identity, + Av1TransformSize.Size64x64 or Av1TransformSize.Size32x64 or Av1TransformSize.Size64x32 or + Av1TransformSize.Size16x64 or Av1TransformSize.Size64x16 => this.TransformType == Av1TransformType.DctDct, _ => true, }; + } /// /// Derives the axis traversal directions encoded by a compound transform type. diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index fce719c422..5971b84f0c 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -10,6 +10,11 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1ForwardTransformTests { + /// + /// Gets every normative transform size, type, and bit-depth combination exercised by the forward and inverse suites. + /// + public static TheoryData ValidTransformCases { get; } = CreateValidTransformCases(); + [Fact] public void DctOperatorsProduceIdenticalScalarAndSimdResults() { @@ -37,21 +42,24 @@ public class Av1ForwardTransformTests AssertOperatorParity(32); } - [Fact] - public void Dct8TwoDimensionalSimdKernelsMatchScalar() - => AssertTransform2dParity(Av1TransformType.DctDct, Av1TransformSize.Size8x8); - - [Fact] - public void Adst16TwoDimensionalSimdKernelsMatchScalarWithBothFlips() - => AssertTransform2dParity(Av1TransformType.FlipAdstFlipAdst, Av1TransformSize.Size16x16); - - [Fact] - public void RectangularDctTwoDimensionalSimdKernelsMatchScalar() - => AssertTransform2dParity(Av1TransformType.DctDct, Av1TransformSize.Size16x8); - - [Fact] - public void Identity32TwoDimensionalSimdKernelsMatchScalar() - => AssertTransform2dParity(Av1TransformType.Identity, Av1TransformSize.Size32x32); + /// + /// Verifies that every applicable SIMD traversal produces the same coefficients as the scalar traversal. + /// + /// The integral value. + /// The integral value. + /// The coded sample bit depth. + [Theory] + [MemberData(nameof(ValidTransformCases))] + public void TwoDimensionalSimdKernelsMatchScalarForEveryValidConfiguration( + int transformTypeValue, + int transformSizeValue, + int bitDepth) + { + Av1TransformType transformType = (Av1TransformType)transformTypeValue; + Av1TransformSize transformSize = (Av1TransformSize)transformSizeValue; + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); + DispatchColumn(transformType, transformSize, bitDepth, ref config); + } [Fact] public void TransformDispatchDoesNotAllocatePerBlock() @@ -138,34 +146,212 @@ public class Av1ForwardTransformTests } } - private static void AssertTransform2dParity(Av1TransformType transformType, Av1TransformSize transformSize) + /// + /// Creates the complete normative transform matrix shared by the forward and inverse parity tests. + /// + /// The transform type, size, and bit-depth cases. + private static TheoryData CreateValidTransformCases() + { + TheoryData cases = []; + + for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) + { + for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) + { + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, 8); + + if (!config.IsAllowed()) + { + continue; + } + + // libaom verifies the low-bit-depth kernel separately from its 10- and 12-bit kernels. Keeping each + // depth as a distinct case makes any fixed-point range failure identify the exact configuration. + for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) + { + cases.Add((int)transformType, (int)transformSize, bitDepth); + } + } + } + + return cases; + } + + /// + /// Closes the static-generic column operator selected by a transform configuration. + /// + /// The compound transform type. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The forward transform configuration. + private static void DispatchColumn( + Av1TransformType transformType, + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) + { + switch (config.TransformFunctionTypeColumn) + { + case Av1TransformFunctionType.Dct4: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct8: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct16: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct32: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct64: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst4: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst8: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst16: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity4: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity8: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity16: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity32: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + default: + Assert.Fail($"Unexpected column function {config.TransformFunctionTypeColumn} for {transformType} {transformSize}."); + break; + } + } + + /// + /// Closes the static-generic row operator after the column operator has been selected. + /// + /// The selected column operator. + /// The compound transform type. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The forward transform configuration. + private static void DispatchRow( + Av1TransformType transformType, + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) + where TColumnOperator : struct, IAv1Transform1dOperator + { + switch (config.TransformFunctionTypeRow) + { + case Av1TransformFunctionType.Dct4: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct8: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct16: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct32: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct64: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst4: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst8: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst16: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity4: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity8: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity16: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity32: + AssertTransform2dParity(transformSize, bitDepth, ref config); + break; + default: + Assert.Fail($"Unexpected row function {config.TransformFunctionTypeRow} for {transformType} {transformSize}."); + break; + } + } + + /// + /// Compares scalar and SIMD forward traversals using padded rows and bounded extreme residuals. + /// + /// The selected column operator. + /// The selected row operator. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The forward transform configuration. + private static void AssertTransform2dParity( + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { - const int bitDepth = 10; int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); - short[] input = new short[width * height]; + int inputStride = width + 5; + int maximum = (1 << bitDepth) - 1; + short[] input = new short[inputStride * height]; - for (int index = 0; index < input.Length; index++) + // Padded rows exercise the same edge-block layout used by the encoder. The alternating extrema are the + // bounded residual limits used by libaom's SIMD match tests and expose wrapping errors in fixed-point stages. + for (int row = 0; row < height; row++) { - input[index] = (short)(((index * 73) % 1023) - 511); + for (int column = 0; column < width; column++) + { + int index = (row * width) + column; + input[(row * inputStride) + column] = (short)((index & 3) switch + { + 0 => maximum, + 1 => -maximum, + 2 => ((index * 73) % ((maximum * 2) + 1)) - maximum, + _ => 0, + }); + } } - Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); - int[] scalar = new int[input.Length]; - int[] vector128 = new int[input.Length]; - int[] vector256 = new int[input.Length]; - int[] scalarWorkspace = new int[Av1TransformWorkspace.MaximumLength]; - int[] vector128Workspace = new int[Av1TransformWorkspace.MaximumLength]; - int[] vector256Workspace = new int[Av1TransformWorkspace.MaximumLength]; + int coefficientCount = width * height; + int workspaceLength = Av1TransformWorkspace.GetRequiredLength(transformSize); + int[] scalar = new int[coefficientCount]; + int[] vector128 = new int[coefficientCount]; + int[] scalarWorkspace = new int[workspaceLength]; + int[] vector128Workspace = new int[workspaceLength]; - Av1ForwardTransformer.Transform2dScalar(input, scalar, (uint)width, ref config, scalarWorkspace); - Av1ForwardTransformer.Transform2dVector128(input, vector128, (uint)width, ref config, vector128Workspace); - Av1ForwardTransformer.Transform2dVector256(input, vector256, (uint)width, ref config, vector256Workspace); + Av1ForwardTransformer.Transform2dScalar(input, scalar, (uint)inputStride, ref config, scalarWorkspace); + Av1ForwardTransformer.Transform2dVector128(input, vector128, (uint)inputStride, ref config, vector128Workspace); Assert.Equal(scalar, vector128); - Assert.Equal(scalar, vector256); + + // The production dispatcher uses 256-bit lanes only when both axes contain a complete eight-lane tile. + if (width >= Vector256.Count && height >= Vector256.Count) + { + int[] vector256 = new int[coefficientCount]; + int[] vector256Workspace = new int[workspaceLength]; + Av1ForwardTransformer.Transform2dVector256(input, vector256, (uint)inputStride, ref config, vector256Workspace); + + Assert.Equal(scalar, vector256); + } } private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 1023) - 511; diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 59e965bf28..2f94b70191 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -113,21 +113,24 @@ public class Av1InverseTransformTests AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); } - [Fact] - public void Dct8TwoDimensionalByteSimdKernelsMatchScalar() - => AssertByteTransform2dParity(Av1TransformType.DctDct, Av1TransformSize.Size8x8); - - [Fact] - public void Adst16TwoDimensionalByteSimdKernelsMatchScalarWithBothFlips() - => AssertByteTransform2dParity(Av1TransformType.FlipAdstFlipAdst, Av1TransformSize.Size16x16); - - [Fact] - public void RectangularDctTwoDimensionalByteSimdKernelsMatchScalar() - => AssertByteTransform2dParity(Av1TransformType.DctDct, Av1TransformSize.Size16x8); - - [Fact] - public void Identity32TwoDimensionalHighBitDepthSimdKernelsMatchScalar() - => AssertHighBitDepthTransform2dParity(Av1TransformType.Identity, Av1TransformSize.Size32x32); + /// + /// Verifies that every applicable SIMD traversal reconstructs the same samples as the scalar traversal. + /// + /// The integral value. + /// The integral value. + /// The coded sample bit depth. + [Theory] + [MemberData(nameof(Av1ForwardTransformTests.ValidTransformCases), MemberType = typeof(Av1ForwardTransformTests))] + public void TwoDimensionalSimdKernelsMatchScalarForEveryValidConfiguration( + int transformTypeValue, + int transformSizeValue, + int bitDepth) + { + Av1TransformType transformType = (Av1TransformType)transformTypeValue; + Av1TransformSize transformSize = (Av1TransformSize)transformSizeValue; + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, bitDepth); + DispatchColumn(transformType, transformSize, bitDepth, ref config); + } [Fact] public void ReconstructionDispatchDoesNotAllocatePerBlock() @@ -304,88 +307,293 @@ public class Av1InverseTransformTests } } - private static void AssertByteTransform2dParity(Av1TransformType transformType, Av1TransformSize transformSize) + /// + /// Closes the static-generic inverse column operator selected by a transform configuration. + /// + /// The compound transform type. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The inverse transform configuration. + private static void DispatchColumn( + Av1TransformType transformType, + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) + { + switch (config.TransformFunctionTypeColumn) + { + case Av1TransformFunctionType.Dct4: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct8: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct16: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct32: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct64: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst4: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst8: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst16: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity4: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity8: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity16: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity32: + DispatchRow(transformType, transformSize, bitDepth, ref config); + break; + default: + Assert.Fail($"Unexpected column function {config.TransformFunctionTypeColumn} for {transformType} {transformSize}."); + break; + } + } + + /// + /// Closes the static-generic inverse row operator after the column operator has been selected. + /// + /// The selected inverse column operator. + /// The compound transform type. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The inverse transform configuration. + private static void DispatchRow( + Av1TransformType transformType, + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) + where TColumnOperator : struct, IAv1Transform1dOperator + { + switch (config.TransformFunctionTypeRow) + { + case Av1TransformFunctionType.Dct4: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct8: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct16: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct32: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Dct64: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst4: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst8: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Adst16: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity4: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity8: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity16: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + case Av1TransformFunctionType.Identity32: + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + break; + default: + Assert.Fail($"Unexpected row function {config.TransformFunctionTypeRow} for {transformType} {transformSize}."); + break; + } + } + + /// + /// Produces bounded conformant coefficients and selects byte or high-bit-depth reconstruction verification. + /// + /// The selected inverse column operator. + /// The selected inverse row operator. + /// The compound transform type. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The inverse transform configuration. + private static void AssertTransform2dParity( + Av1TransformType transformType, + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int inputStride = width + 5; + int maximum = (1 << bitDepth) - 1; + short[] residual = new short[inputStride * height]; + + for (int row = 0; row < height; row++) + { + for (int column = 0; column < width; column++) + { + int index = (row * width) + column; + residual[(row * inputStride) + column] = (short)((index & 3) switch + { + 0 => maximum, + 1 => -maximum, + 2 => ((index * 73) % ((maximum * 2) + 1)) - maximum, + _ => 0, + }); + } + } + + // A conformant forward transform supplies coefficient magnitudes at the exact fixed-point bounds expected by + // the inverse kernels. This is stronger than arbitrary small coefficients and avoids impossible stress inputs. + int[] coefficients = new int[width * height]; + int[] forwardWorkspace = new int[Av1TransformWorkspace.GetRequiredLength(transformSize)]; + Av1ForwardTransformer.Transform2d(residual, coefficients, (uint)inputStride, transformType, transformSize, bitDepth, forwardWorkspace); + + if (bitDepth == 8) + { + AssertByteTransform2dParity(coefficients, transformSize, ref config); + return; + } + + AssertHighBitDepthTransform2dParity(coefficients, transformSize, bitDepth, ref config); + } + + /// + /// Compares eight-bit scalar and SIMD reconstruction with independently padded read and write rows. + /// + /// The selected inverse column operator. + /// The selected inverse row operator. + /// The conformant forward-transform coefficients. + /// The transform-block dimensions. + /// The inverse transform configuration. + private static void AssertByteTransform2dParity( + int[] coefficients, + Av1TransformSize transformSize, + ref Av1Transform2dFlipConfiguration config) where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { const int bitDepth = 8; int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); - int[] coefficients = CreateCoefficients(width * height); - byte[] prediction = new byte[coefficients.Length]; + int readStride = width + 3; + int writeStride = width + 7; + int workspaceLength = Av1TransformWorkspace.GetRequiredLength(transformSize); + byte[] prediction = new byte[readStride * height]; - for (int index = 0; index < prediction.Length; index++) + for (int row = 0; row < height; row++) { - prediction[index] = (byte)(64 + ((index * 29) % 128)); + for (int column = 0; column < width; column++) + { + prediction[(row * readStride) + column] = (byte)(((row * width) + column) * 29); + } } - byte[] scalar = new byte[prediction.Length]; - byte[] vector128 = new byte[prediction.Length]; - byte[] vector256 = new byte[prediction.Length]; - int[] scalarWorkspace = new int[Av1TransformWorkspace.MaximumLength]; - int[] vector128Workspace = new int[Av1TransformWorkspace.MaximumLength]; - int[] vector256Workspace = new int[Av1TransformWorkspace.MaximumLength]; - Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, bitDepth); + byte[] scalar = new byte[writeStride * height]; + byte[] vector128 = new byte[writeStride * height]; + int[] scalarWorkspace = new int[workspaceLength]; + int[] vector128Workspace = new int[workspaceLength]; + Array.Fill(scalar, byte.MaxValue); + Array.Fill(vector128, byte.MaxValue); Av1Inverse2dTransformer.Transform2dScalar( - coefficients, prediction, width, scalar, width, ref config, scalarWorkspace, bitDepth); + coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); Av1Inverse2dTransformer.Transform2dVector128( - coefficients, prediction, width, vector128, width, ref config, vector128Workspace, bitDepth); - - Av1Inverse2dTransformer.Transform2dVector256( - coefficients, prediction, width, vector256, width, ref config, vector256Workspace, bitDepth); + coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); - Assert.Equal(scalar, vector256); + + if (width >= Vector256.Count && height >= Vector256.Count) + { + byte[] vector256 = new byte[writeStride * height]; + int[] vector256Workspace = new int[workspaceLength]; + Array.Fill(vector256, byte.MaxValue); + + Av1Inverse2dTransformer.Transform2dVector256( + coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); + + Assert.Equal(scalar, vector256); + } } - private static void AssertHighBitDepthTransform2dParity(Av1TransformType transformType, Av1TransformSize transformSize) + /// + /// Compares high-bit-depth scalar and SIMD reconstruction with independently padded read and write rows. + /// + /// The selected inverse column operator. + /// The selected inverse row operator. + /// The conformant forward-transform coefficients. + /// The transform-block dimensions. + /// The coded sample bit depth. + /// The inverse transform configuration. + private static void AssertHighBitDepthTransform2dParity( + int[] coefficients, + Av1TransformSize transformSize, + int bitDepth, + ref Av1Transform2dFlipConfiguration config) where TColumnOperator : struct, IAv1Transform1dOperator where TRowOperator : struct, IAv1Transform1dOperator { - const int bitDepth = 10; int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); - int[] coefficients = CreateCoefficients(width * height); - short[] prediction = new short[coefficients.Length]; + int readStride = width + 3; + int writeStride = width + 7; + int maximum = (1 << bitDepth) - 1; + int workspaceLength = Av1TransformWorkspace.GetRequiredLength(transformSize); + short[] prediction = new short[readStride * height]; - for (int index = 0; index < prediction.Length; index++) + for (int row = 0; row < height; row++) { - prediction[index] = (short)(256 + ((index * 47) % 512)); + for (int column = 0; column < width; column++) + { + prediction[(row * readStride) + column] = (short)((((row * width) + column) * 47) & maximum); + } } - short[] scalar = new short[prediction.Length]; - short[] vector128 = new short[prediction.Length]; - short[] vector256 = new short[prediction.Length]; - int[] scalarWorkspace = new int[Av1TransformWorkspace.MaximumLength]; - int[] vector128Workspace = new int[Av1TransformWorkspace.MaximumLength]; - int[] vector256Workspace = new int[Av1TransformWorkspace.MaximumLength]; - Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, bitDepth); + short[] scalar = new short[writeStride * height]; + short[] vector128 = new short[writeStride * height]; + int[] scalarWorkspace = new int[workspaceLength]; + int[] vector128Workspace = new int[workspaceLength]; + Array.Fill(scalar, short.MinValue); + Array.Fill(vector128, short.MinValue); Av1Inverse2dTransformer.Transform2dScalar( - coefficients, prediction, width, scalar, width, ref config, scalarWorkspace, bitDepth); + coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); Av1Inverse2dTransformer.Transform2dVector128( - coefficients, prediction, width, vector128, width, ref config, vector128Workspace, bitDepth); - - Av1Inverse2dTransformer.Transform2dVector256( - coefficients, prediction, width, vector256, width, ref config, vector256Workspace, bitDepth); + coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); - Assert.Equal(scalar, vector256); - } - - private static int[] CreateCoefficients(int length) - { - int[] coefficients = new int[length]; - for (int index = 0; index < coefficients.Length; index++) + if (width >= Vector256.Count && height >= Vector256.Count) { - coefficients[index] = ((index * 37) % 129) - 64; - } + short[] vector256 = new short[writeStride * height]; + int[] vector256Workspace = new int[workspaceLength]; + Array.Fill(vector256, short.MinValue); + + Av1Inverse2dTransformer.Transform2dVector256( + coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); - return coefficients; + Assert.Equal(scalar, vector256); + } } private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 1023) - 511;