diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index febe2f8b9c..7a5b2ed474 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -826,7 +826,7 @@ Encoder verification contract: - [~] Finalized transform coefficients and packed EOB/type state now use raster-ordered, per-superblock plane segments matching current libaom's coefficient-pool geometry. One ImageSharp allocator owner replaces libaom's separate coefficient, EOB, and entropy-context allocations while preserving the full 1024 luma and 256-per-chroma 4x4 state capacity of a 128x128 4:2:0 superblock; the forward transform and mode-decision stages still need to populate this owner. - [~] Tile partition writing now follows current libaom's recursive `write_modes_sb` preorder traversal and `update_ext_partition_context` edge updates directly. The obsolete SVT-derived global geometry catalog and its unimplemented lookup are removed; transform geometry is derived in libaom's bounded 64x64 residual order, fixed intra transform-size symbols use the reference depth and neighbor contexts, frame-edge and segmentation syntax use mode-information units, and 128x128 CDEF units use libaom's 0-to-3 indexing and first-block strength ownership. Partition and mode analysis still need to populate these retained decisions; variable inter-transform syntax remains part of later inter-frame support. - [ ] Implement legal deblocking, CDEF, restoration, super-resolution, and film-grain signaling decisions. -- [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform. Its range coder matches current libaom's 64-bit coding window, bulk big-endian byte flush, and backward carry propagation while using one byte of allocator scratch per estimated output byte instead of the former 16-bit pre-carry storage. The reference-type symbol encoder is passed normally through tile traversal, and the operation boundary owns the allocator-backed item payload stream for exactly one synchronous encode. Every remaining encoder fragment must be audited before it becomes active. +- [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform, defers both coefficient rents until the first nonzero transform block, and disposes all tile scratch independently from the detached encoded bytes. Its range coder matches current libaom's 64-bit coding window, bulk big-endian byte flush, and backward carry propagation while using one byte of allocator scratch per estimated output byte instead of the former 16-bit pre-carry storage. The reference-type symbol encoder is passed normally through tile traversal, and the operation boundary owns the allocator-backed item payload stream for exactly one synchronous encode. Every remaining encoder fragment must be audited before it becomes active. - [~] The planar conversion, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Apply the same rule to every later hot-path family. - [~] Forward-quantizer FeatureTestRunner and zero-allocation tests compare every hardware tier with an independent scan-order scalar oracle shaped from current-main libaom. Both passed direct net11 VSTest in Release. - [~] The combined-frame writer now completes the byte-counted uncompressed frame header before starting the optional multi-tile tile-group flag, matching current libaom's separate frame-header and tile-group writers. A non-uniform two-tile round trip verifies the explicit boundaries, both tile payloads, and complete stream consumption through direct net11 VSTest in Release. diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs index 30ba8621a0..4b8e5272ff 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs @@ -141,15 +141,20 @@ internal class Av1SymbolEncoder : IDisposable /// private bool isDisposed; + /// + /// The configuration providing lazily allocated coefficient scratch. + /// + private readonly Configuration configuration; + /// /// The reusable padded coefficient levels used to derive entropy contexts. /// - private readonly Av1LevelBuffer levels; + private Av1LevelBuffer? levels; /// /// The reusable raster-order coefficient contexts for one transform. /// - private readonly IMemoryOwner coefficientContexts; + private IMemoryOwner? coefficientContexts; /// /// The range writer producing the current tile payload. @@ -170,6 +175,8 @@ internal class Av1SymbolEncoder : IDisposable /// A value indicating whether encoded symbols adapt their tile distributions. public Av1SymbolEncoder(Configuration configuration, int initialSize, int qIndex, bool updateCdf = true) { + this.configuration = configuration; + // Every default accessor creates independently mutable state. Encoding and decoding therefore begin from // equivalent tile-local models without constructing and immediately deep-copying a second object graph. this.tileIntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy; @@ -194,8 +201,6 @@ internal class Av1SymbolEncoder : IDisposable this.coefficientsBaseEndOfBlock = Av1DefaultDistributions.GetBaseEndOfBlock(qIndex); this.dcSign = Av1DefaultDistributions.GetDcSign(qIndex); this.endOfBlockExtra = Av1DefaultDistributions.GetEndOfBlockExtra(qIndex); - this.levels = new(configuration); - this.coefficientContexts = configuration.MemoryAllocator.Allocate(MaximumCoefficientContextCount); this.writer = new(configuration, initialSize, updateCdf); this.baseQIndex = qIndex; } @@ -290,14 +295,6 @@ internal class Av1SymbolEncoder : IDisposable ReadOnlySpan scan = scanOrder.Scan; Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize); - ref Av1SymbolWriter w = ref this.writer; - - // AV1 omits high-frequency coefficients beyond 32 samples on every 64-point transform dimension. The tile - // owns maximum-sized workspaces so repeated transform coding changes only their active views. - this.levels.Reset(new Size(width, height)); - Span coefficientContexts = this.coefficientContexts.Memory.Span[..(width * height)]; - coefficientContexts.Clear(); - Guard.MustBeLessThan((int)transformSizeContext, (int)Av1TransformSize.AllSizes, nameof(transformSizeContext)); this.WriteTransformBlockSkip(endOfBlock == 0, transformSizeContext, transformBlockContext.SkipContext); @@ -307,7 +304,17 @@ internal class Av1SymbolEncoder : IDisposable return 0; } - this.levels.Initialize(coefficientBuffer); + ref Av1SymbolWriter w = ref this.writer; + Av1LevelBuffer levels = this.levels ??= new(this.configuration); + IMemoryOwner coefficientContextOwner = this.coefficientContexts ??= + this.configuration.MemoryAllocator.Allocate(MaximumCoefficientContextCount); + + // AV1 omits high-frequency coefficients beyond 32 samples on every 64-point transform dimension. The tile + // creates maximum-sized workspaces only when nonzero coefficient syntax needs them, then changes only their active views. + levels.Reset(new Size(width, height)); + Span coefficientContexts = coefficientContextOwner.Memory.Span[..(width * height)]; + coefficientContexts.Clear(); + levels.Initialize(coefficientBuffer); if (componentType == Av1ComponentType.Luminance) { this.WriteTransformType(transformType, transformSize, useReducedTransformSet, this.baseQIndex, filterIntraMode, intraDirection); @@ -315,14 +322,14 @@ internal class Av1SymbolEncoder : IDisposable this.WriteEndOfBlockPosition(endOfBlock, componentType, transformClass, transformSize, transformSizeContext); - Av1SymbolContextHelper.GetNzMapContexts(this.levels, scan, endOfBlock, transformSize, transformClass, coefficientContexts); + Av1SymbolContextHelper.GetNzMapContexts(levels, scan, endOfBlock, transformSize, transformClass, coefficientContexts); int limitedTransformSizeContext = Math.Min((int)transformSizeContext, (int)Av1TransformSize.Size32x32); for (c = endOfBlock - 1; c >= 0; --c) { short pos = scan[c]; int v = coefficientBuffer[pos]; short coeffContext = coefficientContexts[pos]; - Point position = this.levels.GetPosition(pos); + Point position = levels.GetPosition(pos); int level = Math.Abs(v); if (c == endOfBlock - 1) @@ -338,7 +345,7 @@ internal class Av1SymbolEncoder : IDisposable { // Base-range symbols extend levels above the two base levels in fixed-size chunks. int baseRange = level - 1 - Av1Constants.BaseLevelsCount; - int baseRangeContext = Av1SymbolContextHelper.GetBaseRangeContext(this.levels, position, transformClass); + int baseRangeContext = Av1SymbolContextHelper.GetBaseRangeContext(levels, position, transformClass); for (int idx = 0; idx < Av1Constants.CoefficientBaseRange; idx += Av1Constants.BaseRangeSizeMinus1) { int k = Math.Min(baseRange - idx, Av1Constants.BaseRangeSizeMinus1); @@ -478,8 +485,8 @@ internal class Av1SymbolEncoder : IDisposable { if (!this.isDisposed) { - this.coefficientContexts.Dispose(); - this.levels.Dispose(); + this.coefficientContexts?.Dispose(); + this.levels?.Dispose(); this.writer.Dispose(); this.isDisposed = true; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs index fa5411dcd2..f3d0c874a7 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs @@ -295,7 +295,7 @@ public class Av1CoefficientsEntropyTests transformBlocks.Fill(new Av1EncoderTransformBlockState { TransformType = Av1TransformType.Identity }); Av1EncoderBlockStruct block = default; - Av1SymbolEncoder writer = new(Configuration.Default, 4096, BaseQIndex); + using Av1SymbolEncoder writer = new(Configuration.Default, 4096, BaseQIndex); Av1TileWriter.EncodeTransformCoefficientsY( picture, context, @@ -379,7 +379,7 @@ public class Av1CoefficientsEntropyTests transforms.Left[leftIndex] = 16; picture.TransformFunctionContexts = [transforms]; - Av1SymbolEncoder writer = new(Configuration.Default, 64, BaseQIndex); + using Av1SymbolEncoder writer = new(Configuration.Default, 64, BaseQIndex); Av1TileWriter.WriteTransformSize( picture, writer, @@ -631,7 +631,7 @@ public class Av1CoefficientsEntropyTests width: 128, height: 64); - Av1SymbolEncoder writer = new(Configuration.Default, 512, BaseQIndex); + using Av1SymbolEncoder writer = new(Configuration.Default, 512, BaseQIndex); Av1TileWriter.WriteSuperblock( picture, @@ -683,7 +683,7 @@ public class Av1CoefficientsEntropyTests int[] leftContexts = new int[1]; Av1TransformBlockContext transformBlockContext = default; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Span coefficientsBuffer = [1, 2, 3, 4, 5]; Span expected = new int[16]; Span actuals = new int[16]; @@ -753,7 +753,7 @@ public class Av1CoefficientsEntropyTests int[] leftContexts = new int[1]; Av1TransformBlockContext transformBlockContext = default; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Span coefficientsBuffer = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; Span actuals = new int[16 + 1]; @@ -828,7 +828,7 @@ public class Av1CoefficientsEntropyTests int[] leftContexts = new int[transformSize.Get4x4HighCount()]; Av1TransformBlockContext transformBlockContext = default; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Span coefficientsBuffer = Enumerable.Range(0, blockSize.GetHeight() * blockSize.GetWidth()).ToArray(); Span actuals = new int[16 + 1]; diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs index eaf34fa525..35c6c3a2c6 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs @@ -55,6 +55,67 @@ public class Av1EntropyTests Assert.Equal(allocation.HashCodeOfBuffer, returned.HashCodeOfBuffer); } + [Fact] + public void SymbolEncoderRentsCoefficientScratchOnlyForNonzeroBlocks() + { + TestMemoryAllocator allocator = new(); + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + Span coefficients = stackalloc int[16]; + + using (Av1SymbolEncoder encoder = new(configuration, 64, BaseQIndex)) + { + TestMemoryAllocator.AllocationRequest outputScratch = Assert.Single(allocator.AllocationLog); + Assert.Equal(typeof(byte), outputScratch.ElementType); + + int emptyContext = encoder.WriteCoefficients( + Av1TransformSize.Size4x4, + Av1TransformType.DctDct, + Av1PredictionMode.DC, + coefficients, + Av1ComponentType.Luminance, + default, + 0, + false, + Av1FilterIntraMode.DC); + + Assert.Equal(0, emptyContext); + Assert.Single(allocator.AllocationLog); + + coefficients[0] = 1; + encoder.WriteCoefficients( + Av1TransformSize.Size4x4, + Av1TransformType.DctDct, + Av1PredictionMode.DC, + coefficients, + Av1ComponentType.Luminance, + default, + 1, + false, + Av1FilterIntraMode.DC); + + Assert.Equal(3, allocator.AllocationLog.Count); + TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[1]; + TestMemoryAllocator.AllocationRequest contextScratch = allocator.AllocationLog[2]; + int maximumTransformDimension = Av1Constants.MaxTransformSize / 2; + int expectedLevelLength = + (Av1Constants.TransformPadHorizontal + maximumTransformDimension) * + (Av1Constants.TransformPadTop + maximumTransformDimension + Av1Constants.TransformPadBottom); + + Assert.Equal(typeof(byte), levelScratch.ElementType); + Assert.Equal(expectedLevelLength, levelScratch.Length); + Assert.Equal(AllocationOptions.Clean, levelScratch.AllocationOptions); + Assert.Equal(typeof(sbyte), contextScratch.ElementType); + Assert.Equal(maximumTransformDimension * maximumTransformDimension, contextScratch.Length); + } + + Assert.Equal(3, allocator.ReturnLog.Count); + Assert.Equal( + allocator.AllocationLog.Select(x => x.AllocationId).Order(), + allocator.ReturnLog.Select(x => x.AllocationId).Order()); + } + [Fact] public void ReadRandomLiteral() { @@ -236,7 +297,7 @@ public class Av1EntropyTests { // Assign Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Av1PartitionType[] values = [ Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.None, Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.None, Av1PartitionType.None]; @@ -267,7 +328,7 @@ public class Av1EntropyTests // Assign Av1BlockSize blockSize = (Av1BlockSize)size; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Av1PartitionType[] values = [ Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Horizontal, Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Horizontal, Av1PartitionType.Horizontal]; @@ -298,7 +359,7 @@ public class Av1EntropyTests // Assign Av1BlockSize blockSize = (Av1BlockSize)size; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Av1PartitionType[] values = [ Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Vertical, Av1PartitionType.Split, Av1PartitionType.Split, Av1PartitionType.Vertical, Av1PartitionType.Vertical]; @@ -330,7 +391,7 @@ public class Av1EntropyTests { // Assign Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); bool[] values = [true, true, false, false, false, false, false, false, true]; bool[] actuals = new bool[values.Length]; @@ -359,7 +420,7 @@ public class Av1EntropyTests // Assign Av1TransformSize transformSizeContext = (Av1TransformSize)transformContext; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); bool[] values = [true, true, false, false, false, false, false, false, true]; bool[] actuals = new bool[values.Length]; @@ -392,7 +453,7 @@ public class Av1EntropyTests Av1FilterIntraMode filterIntraMode = (Av1FilterIntraMode)intraMode; Av1PredictionMode intraDirection = (Av1PredictionMode)intraDir; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); // TODO: Include AdstFlipAdst, which is currently mapped to Identity. Av1TransformType[] values = [ @@ -430,7 +491,7 @@ public class Av1EntropyTests Av1PlaneType planeType = (Av1PlaneType)plane; Av1TransformClass transformClass = (Av1TransformClass)txClass; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); int[] values = [1, 2, 3, 4, 5]; int[] actuals = new int[values.Length]; @@ -458,7 +519,7 @@ public class Av1EntropyTests { // Assign Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); int[] values = Enumerable.Range(0, 16384).ToArray(); int[] actuals = new int[values.Length]; @@ -490,7 +551,7 @@ public class Av1EntropyTests // Assign int[] values = [3, 6, 7, 0, 2, 0, 2, 1, 1]; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); int[] actuals = new int[values.Length]; // Act @@ -517,7 +578,7 @@ public class Av1EntropyTests // Assign int[] values = [3, 6, -7, -8, -2, 0, 2, 1, -1]; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); int[] actuals = new int[values.Length]; // Act @@ -548,7 +609,7 @@ public class Av1EntropyTests Av1FilterIntraMode.DC, Av1FilterIntraMode.Vertical, Av1FilterIntraMode.DC, Av1FilterIntraMode.Paeth, Av1FilterIntraMode.AllFilterIntraModes, Av1FilterIntraMode.Directional157, Av1FilterIntraMode.DC, Av1FilterIntraMode.Directional157]; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); Av1FilterIntraMode[] actuals = new Av1FilterIntraMode[values.Length]; // Act @@ -575,7 +636,7 @@ public class Av1EntropyTests // Assign bool[] values = [true, true, false, true, false, false, false]; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex); bool[] actuals = new bool[values.Length]; // Act @@ -621,7 +682,7 @@ public class Av1EntropyTests ]; Configuration configuration = Configuration.Default; - Av1SymbolEncoder encoder = new(configuration, 64, BaseQIndex); + using Av1SymbolEncoder encoder = new(configuration, 64, BaseQIndex); for (int i = 0; i < values.Length; i++) {