From 08406d635d86f1df3b738b098763c569a15590db Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 2 Sep 2026 14:23:15 +1000 Subject: [PATCH] Compose AV1 transform block encoding --- HEIF_IMPLEMENTATION_PLAN.md | 5 +- .../Av1/Pipeline/Av1TransformBlockEncoder.cs | 75 +++++++ .../Heif/Av1/Transform/Av1CoefficientShape.cs | 30 --- .../Transform/Av1ForwardTransformerFactory.cs | 201 ------------------ .../Heif/Av1/Av1TransformBlockEncoderTests.cs | 173 +++++++++++++++ 5 files changed, 251 insertions(+), 233 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1CoefficientShape.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformerFactory.cs create mode 100644 tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index a919951b59..30a8698a7a 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -811,7 +811,7 @@ Encoder verification contract: ### 6. Build the complete AV1 frame encoder - [~] SIMD-first RGB-to-native-plane conversion now feeds eight-bit and high-bit-depth bordered AV1 source frames directly, preserving ImageSharp's arbitrary packed-pixel input contract without an intermediate full-frame native-plane copy. -- [~] Forward transform families and transform workspace exist locally. +- [~] Forward transform families, transform workspace, and a zero-allocation transform-then-quantize block boundary exist locally. The block boundary matches current libaom's selected full-transform, quantization, qcoeff, dqcoeff, EOB, and transform-type data flow; frame traversal still needs to select blocks and supply coefficient-owner slices. - [~] Symbol writer, coefficient writer, and tile writer fragments exist locally. - [~] A non-owning encoder-frame view now separates visible conversion regions from coded regions and performs complete left, top, right, bottom, and corner extension across each bordered plane. Current libaom uses 8-sample-aligned coded dimensions, a 32-sample-aligned luma stride with chroma stride derived from it, and a 64-pixel luma border for non-resized all-intra encoding. The frame-encoder boundary now converts packed pixels directly into those final source planes before extension; the containing encode operation still needs to connect matching source and reconstruction plane rents with ordinary `using` lifetimes. - [~] Temporal delimiter, sequence header, frame header, and combined-frame tile-group writing exist locally. The remaining required metadata, padding, and encoder-wide syntax paths are not complete. @@ -823,12 +823,13 @@ Encoder verification contract: - [~] The tile writer now publishes one packed coefficient context per covered 4x4 edge unit and derives luma/chroma skip plus DC-sign contexts from the complete transform edges using current-libaom units. Complete tile traversal, initialized picture state, and verified CDF update behavior remain. - [~] Encoder mode information now uses a frame-owned reference grid over its contiguous allocation, matching current libaom's `mi_grid_base` and `mi_alloc` ownership without per-block tail copies. Signed relative neighbor lookup, 4x4-unit addressing, and mutable selected skip syntax have focused contracts; complete mode decision still remains. - [~] Final block decisions now use contiguous value storage with palette and prediction syntax inline. Macroblock edge and neighbor state is reused by the entropy-coding operation instead of allocating one managed object per final block; directional deltas retain their signed range, while complete mode decision still remains. -- [~] Finalized transform coefficients and packed EOB/type state now use raster-ordered, per-superblock plane segments matching current libaom's coefficient-pool geometry. One ImageSharp allocator owner replaces libaom's separate coefficient, EOB, and entropy-context allocations while preserving the full 1024 luma and 256-per-chroma 4x4 state capacity of a 128x128 4:2:0 superblock; the forward transform and mode-decision stages still need to populate this owner. +- [~] Finalized transform coefficients and packed EOB/type state now use raster-ordered, per-superblock plane segments matching current libaom's coefficient-pool geometry. One ImageSharp allocator owner replaces libaom's separate coefficient, EOB, and entropy-context allocations while preserving the full 1024 luma and 256-per-chroma 4x4 state capacity of a 128x128 4:2:0 superblock. The transform-block boundary can populate the owner's quantized coefficient and state slices directly; mode-decision traversal still needs to select and invoke it. - [~] Tile partition writing now follows current libaom's recursive `write_modes_sb` preorder traversal and `update_ext_partition_context` edge updates directly. The obsolete SVT-derived global geometry catalog and its unimplemented lookup are removed; transform geometry is derived in libaom's bounded 64x64 residual order, fixed intra transform-size symbols use the reference depth and neighbor contexts, frame-edge and segmentation syntax use mode-information units, and 128x128 CDEF units use libaom's 0-to-3 indexing and first-block strength ownership. Partition and mode analysis still need to populate these retained decisions; variable inter-transform syntax remains part of later inter-frame support. - [ ] Implement legal deblocking, CDEF, restoration, super-resolution, and film-grain signaling decisions. - [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform, defers both coefficient rents until the first nonzero transform block, and disposes all tile scratch independently from the detached encoded bytes. Its range coder matches current libaom's 64-bit coding window, bulk big-endian byte flush, and backward carry propagation while using one byte of allocator scratch per estimated output byte instead of the former 16-bit pre-carry storage. The reference-type symbol encoder is passed normally through tile traversal, and the operation boundary owns the allocator-backed item payload stream for exactly one synchronous encode. Every remaining encoder fragment must be audited before it becomes active. - [~] The planar conversion, residual construction, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Residual construction matches current libaom's exact source-minus-prediction arithmetic for 8-bit and high-bit-depth planes, preserves independent row strides and unaligned starts, and writes directly into caller-owned signed-short storage without allocation. Apply the same rule to every later hot-path family. - [~] Residual tests verify misaligned planes, independent source, prediction, and destination strides, SIMD remainders, untouched padding, 8-bit, 10-bit, and 12-bit precision, every operator width independently of host acceleration, the scalar fallback, and zero per-transform allocations. +- [~] The unused coefficient-shape transform facade and its unimplemented N2, N4, and DC-only branches are removed. Finalized block encoding now follows the complete-transform path that current libaom uses before fast quantization; later rate-distortion search may add proven coefficient optimization without exposing inactive runtime throws. - [~] Forward-quantizer FeatureTestRunner and zero-allocation tests compare every hardware tier with an independent scan-order scalar oracle shaped from current-main libaom. Both passed direct net11 VSTest in Release. - [~] The combined-frame writer now completes the byte-counted uncompressed frame header before starting the optional multi-tile tile-group flag, matching current libaom's separate frame-header and tile-group writers. A non-uniform two-tile round trip verifies the explicit boundaries, both tile payloads, and complete stream consumption through direct net11 VSTest in Release. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs new file mode 100644 index 0000000000..493b249ad4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs @@ -0,0 +1,75 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Transforms and quantizes finalized AV1 residual blocks. +/// +internal static class Av1TransformBlockEncoder +{ + /// + /// Applies a lossy forward transform and quantization to one residual block. + /// + /// The spatial residual samples. + /// The number of residual samples between rows. + /// The reusable forward-transform coefficient workspace. + /// The retained entropy-coding coefficients. + /// The reusable reconstruction coefficients. + /// The reusable internal transform workspace. + /// The selected transform dimensions. + /// The selected compound transform type. + /// The segment quantizer index. + /// The plane DC quantizer adjustment. + /// The plane AC quantizer adjustment. + /// The coded sample bit depth. + /// The retained transform type and end-of-block syntax. + public static void EncodeLossy( + Span residual, + uint residualStride, + Span transformCoefficients, + Span quantizedCoefficients, + Span dequantizedCoefficients, + Span transformWorkspace, + Av1TransformSize transformSize, + Av1TransformType transformType, + int qIndex, + int dcDeltaQ, + int acDeltaQ, + Av1BitDepth bitDepth, + ref Av1EncoderTransformBlockState state) + { + int coefficientCount = transformSize.GetAdjusted().GetSize2d(); + Span transformed = transformCoefficients[..coefficientCount]; + Span quantized = quantizedCoefficients[..coefficientCount]; + Span dequantized = dequantizedCoefficients[..coefficientCount]; + + // The encoder keeps transformed, quantized, and reconstructed coefficients separate because mode decision + // consumes all three while only the quantized values survive in the frame coefficient owner. + Av1ForwardTransformer.Transform2d( + residual, + transformed, + residualStride, + transformType, + transformSize, + bitDepth.GetBitCount(), + transformWorkspace); + + state.EndOfBlock = Av1ForwardQuantizer.QuantizeLossy( + transformed, + quantized, + dequantized, + transformSize, + transformType, + qIndex, + dcDeltaQ, + acDeltaQ, + bitDepth); + + state.TransformType = transformType; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1CoefficientShape.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1CoefficientShape.cs deleted file mode 100644 index 1cb1492887..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1CoefficientShape.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Identifies how much of a transform coefficient plane the encoder evaluates. -/// -internal enum Av1CoefficientShape -{ - /// - /// Evaluates the complete coefficient plane. - /// - Default, - - /// - /// Evaluates the half-coefficient shape. - /// - N2, - - /// - /// Evaluates the quarter-coefficient shape. - /// - N4, - - /// - /// Evaluates only the DC coefficient. - /// - OnlyDc -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformerFactory.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformerFactory.cs deleted file mode 100644 index 56b89d4424..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformerFactory.cs +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Selects the forward-transform coefficient shape used by AV1 encoder mode decision. -/// -internal static class Av1ForwardTransformerFactory -{ - /// - /// Applies the encoder-selected coefficient-shape transform to a residual block. - /// - /// The spatial residual samples. - /// The number of residual samples between rows. - /// The destination transform coefficients. - /// The number of coefficient positions between rows. - /// The transform-block dimensions. - /// The accumulated energy outside the retained coefficient shape. - /// The source sample bit depth. - /// The compound transform type. - /// The luma or chroma component class. - /// The subset of coefficients evaluated by mode decision. - /// The reusable transform workspace for the containing encode operation. - public static void EstimateTransform( - Span residualBuffer, - uint residualStride, - Span coefficientBuffer, - uint coefficientStride, - Av1TransformSize transformSize, - ref ulong threeQuadEnergy, - int bitDepth, - Av1TransformType transformType, - Av1PlaneType componentType, - Av1CoefficientShape transformCoefficientShape, - Span workspace) - { - switch (transformCoefficientShape) - { - case Av1CoefficientShape.Default: - EstimateTransformDefault( - residualBuffer, - residualStride, - coefficientBuffer, - coefficientStride, - transformSize, - ref threeQuadEnergy, - bitDepth, - transformType, - componentType, - workspace); - - break; - case Av1CoefficientShape.N2: - EstimateTransformN2( - residualBuffer, - residualStride, - coefficientBuffer, - coefficientStride, - transformSize, - ref threeQuadEnergy, - bitDepth, - transformType, - componentType, - workspace); - - break; - case Av1CoefficientShape.N4: - EstimateTransformN4( - residualBuffer, - residualStride, - coefficientBuffer, - coefficientStride, - transformSize, - ref threeQuadEnergy, - bitDepth, - transformType, - componentType, - workspace); - - break; - case Av1CoefficientShape.OnlyDc: - EstimateTransformOnlyDc( - residualBuffer, - residualStride, - coefficientBuffer, - coefficientStride, - transformSize, - ref threeQuadEnergy, - bitDepth, - transformType, - componentType, - workspace); - - break; - } - } - - /// - /// Applies the complete two-dimensional transform without discarding coefficients. - /// - /// The spatial residual samples. - /// The number of residual samples between rows. - /// The destination transform coefficients. - /// The number of coefficient positions between rows. - /// The transform-block dimensions. - /// The accumulated energy outside the retained coefficient shape. - /// The source sample bit depth. - /// The compound transform type. - /// The luma or chroma component class. - /// The reusable transform workspace for the containing encode operation. - private static void EstimateTransformDefault( - Span residualBuffer, - uint residualStride, - Span coefficientBuffer, - uint coefficientStride, - Av1TransformSize transformSize, - ref ulong threeQuadEnergy, - int bitDepth, - Av1TransformType transformType, - Av1PlaneType componentType, - Span workspace) - => Av1ForwardTransformer.Transform2d(residualBuffer, coefficientBuffer, residualStride, transformType, transformSize, bitDepth, workspace); - - /// - /// Applies the half-coefficient transform shape and measures the discarded coefficient energy. - /// - /// The spatial residual samples. - /// The number of residual samples between rows. - /// The destination transform coefficients. - /// The number of coefficient positions between rows. - /// The transform-block dimensions. - /// The accumulated energy outside the retained coefficient shape. - /// The source sample bit depth. - /// The compound transform type. - /// The luma or chroma component class. - /// The reusable transform workspace for the containing encode operation. - private static void EstimateTransformN2( - Span residualBuffer, - uint residualStride, - Span coefficientBuffer, - uint coefficientStride, - Av1TransformSize transformSize, - ref ulong threeQuadEnergy, - int bitDepth, - Av1TransformType transformType, - Av1PlaneType componentType, - Span workspace) => throw new NotImplementedException(); - - /// - /// Applies the quarter-coefficient transform shape and measures the discarded coefficient energy. - /// - /// The spatial residual samples. - /// The number of residual samples between rows. - /// The destination transform coefficients. - /// The number of coefficient positions between rows. - /// The transform-block dimensions. - /// The accumulated energy outside the retained coefficient shape. - /// The source sample bit depth. - /// The compound transform type. - /// The luma or chroma component class. - /// The reusable transform workspace for the containing encode operation. - private static void EstimateTransformN4( - Span residualBuffer, - uint residualStride, - Span coefficientBuffer, - uint coefficientStride, - Av1TransformSize transformSize, - ref ulong threeQuadEnergy, - int bitDepth, - Av1TransformType transformType, - Av1PlaneType componentType, - Span workspace) => throw new NotImplementedException(); - - /// - /// Evaluates only the transform's DC coefficient and measures the discarded coefficient energy. - /// - /// The spatial residual samples. - /// The number of residual samples between rows. - /// The destination transform coefficients. - /// The number of coefficient positions between rows. - /// The transform-block dimensions. - /// The accumulated energy outside the retained coefficient shape. - /// The source sample bit depth. - /// The compound transform type. - /// The luma or chroma component class. - /// The reusable transform workspace for the containing encode operation. - private static void EstimateTransformOnlyDc( - Span residualBuffer, - uint residualStride, - Span coefficientBuffer, - uint coefficientStride, - Av1TransformSize transformSize, - ref ulong threeQuadEnergy, - int bitDepth, - Av1TransformType transformType, - Av1PlaneType componentType, - Span workspace) => throw new NotImplementedException(); -} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs new file mode 100644 index 0000000000..d48a15148c --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs @@ -0,0 +1,173 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies the finalized forward-transform and quantization block boundary. +/// +[Trait("Format", "Avif")] +public class Av1TransformBlockEncoderTests +{ + /// + /// Verifies that the composed block path retains the exact outputs already established for its arithmetic stages. + /// + [Fact] + public void LossyBlockEncodingMatchesTransformAndQuantizerContracts() + { + ValidateBlock(Av1TransformSize.Size4x4, Av1TransformType.DctDct, Av1BitDepth.EightBit, 1); + ValidateBlock(Av1TransformSize.Size8x8, Av1TransformType.Identity, Av1BitDepth.TenBit, 73); + ValidateBlock(Av1TransformSize.Size32x64, Av1TransformType.DctDct, Av1BitDepth.TenBit, 173); + ValidateBlock(Av1TransformSize.Size64x64, Av1TransformType.DctDct, Av1BitDepth.TwelveBit, 255); + } + + /// + /// Verifies that repeated maximum-transform block encoding uses only caller-owned workspaces. + /// + [Fact] + public void LossyBlockEncodingDoesNotAllocate() + { + Av1TransformSize transformSize = Av1TransformSize.Size64x64; + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int coefficientCount = transformSize.GetAdjusted().GetSize2d(); + short[] residual = new short[width * height]; + int[] transformed = new int[coefficientCount]; + int[] quantized = new int[coefficientCount]; + int[] dequantized = new int[coefficientCount]; + int[] workspace = new int[Av1TransformWorkspace.MaximumLength]; + FillResidual(residual, width, width, height, 4095); + Av1EncoderTransformBlockState state = default; + + Av1TransformBlockEncoder.EncodeLossy( + residual, + (uint)width, + transformed, + quantized, + dequantized, + workspace, + transformSize, + Av1TransformType.DctDct, + 73, + -1, + 3, + Av1BitDepth.TwelveBit, + ref state); + + long before = GC.GetAllocatedBytesForCurrentThread(); + for (int iteration = 0; iteration < 16; iteration++) + { + Av1TransformBlockEncoder.EncodeLossy( + residual, + (uint)width, + transformed, + quantized, + dequantized, + workspace, + transformSize, + Av1TransformType.DctDct, + 73, + -1, + 3, + Av1BitDepth.TwelveBit, + ref state); + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + } + + private static void ValidateBlock( + Av1TransformSize transformSize, + Av1TransformType transformType, + Av1BitDepth bitDepth, + int qIndex) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int residualStride = width + 3; + int coefficientCount = transformSize.GetAdjusted().GetSize2d(); + int sampleMaximum = (1 << bitDepth.GetBitCount()) - 1; + short[] residual = new short[residualStride * height]; + int[] expectedTransformed = new int[coefficientCount + 7]; + int[] expectedQuantized = new int[coefficientCount + 7]; + int[] expectedDequantized = new int[coefficientCount + 7]; + int[] actualTransformed = new int[coefficientCount + 7]; + int[] actualQuantized = new int[coefficientCount + 7]; + int[] actualDequantized = new int[coefficientCount + 7]; + int[] expectedWorkspace = new int[Av1TransformWorkspace.MaximumLength]; + int[] actualWorkspace = new int[Av1TransformWorkspace.MaximumLength]; + Array.Fill(expectedTransformed, int.MinValue); + Array.Fill(expectedQuantized, int.MinValue); + Array.Fill(expectedDequantized, int.MinValue); + Array.Fill(actualTransformed, int.MinValue); + Array.Fill(actualQuantized, int.MinValue); + Array.Fill(actualDequantized, int.MinValue); + FillResidual(residual, residualStride, width, height, sampleMaximum); + + Av1ForwardTransformer.Transform2d( + residual, + expectedTransformed.AsSpan(0, coefficientCount), + (uint)residualStride, + transformType, + transformSize, + bitDepth.GetBitCount(), + expectedWorkspace); + + ushort expectedEndOfBlock = Av1ForwardQuantizer.QuantizeLossy( + expectedTransformed, + expectedQuantized, + expectedDequantized, + transformSize, + transformType, + qIndex, + -1, + 3, + bitDepth); + + Av1EncoderTransformBlockState actualState = default; + Av1TransformBlockEncoder.EncodeLossy( + residual, + (uint)residualStride, + actualTransformed, + actualQuantized, + actualDequantized, + actualWorkspace, + transformSize, + transformType, + qIndex, + -1, + 3, + bitDepth, + ref actualState); + + Assert.Equal(expectedTransformed, actualTransformed); + Assert.Equal(expectedQuantized, actualQuantized); + Assert.Equal(expectedDequantized, actualDequantized); + Assert.Equal(expectedEndOfBlock, actualState.EndOfBlock); + Assert.Equal(transformType, actualState.TransformType); + } + + private static void FillResidual(Span residual, int stride, int width, int height, int sampleMaximum) + { + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + int index = (y * width) + x; + residual[(y * stride) + x] = (short)((index & 3) switch + { + 0 => sampleMaximum, + 1 => -sampleMaximum, + 2 => ((index * 73) % ((2 * sampleMaximum) + 1)) - sampleMaximum, + _ => 0, + }); + } + } + } +}