diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index b2be2557ba..2f8f27f738 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -821,7 +821,7 @@ Encoder verification contract: - [~] SIMD-first RGB-to-native-plane conversion now feeds eight-bit and high-bit-depth bordered AV1 source frames directly, preserving ImageSharp's arbitrary packed-pixel input contract without an intermediate full-frame native-plane copy. - [~] Forward transform families, transform workspace, and a zero-allocation transform-then-quantize block boundary exist locally. The block boundary matches current libaom's selected full-transform, quantization, qcoeff, dqcoeff, EOB, and transform-type data flow. One reusable 61 KiB allocator owner supplies tightly packed residual, aligned transform-coefficient, dequantized-coefficient, and transform scratch spans across transform blocks; quantized coefficients write directly to the retained frame coefficient owner instead of being duplicated. The exact owner type, length, span partitioning, and exactly-once return are verified, and the adjacent residual, transform, quantizer, and block tests pass 12 of 12 through direct net11 VSTest in Release. Frame traversal still needs to select blocks and supply coefficient-owner slices. - [~] Symbol writer, coefficient writer, and tile writer fragments exist locally. -- [~] A non-owning encoder-frame view now separates visible conversion regions from coded regions and performs complete left, top, right, bottom, and corner extension across each bordered plane. Current libaom uses 8-sample-aligned coded dimensions, a 32-sample-aligned luma stride with chroma stride derived from it, and a 64-pixel luma border for non-resized all-intra encoding. The frame-encoder boundary now converts packed pixels directly into those final source planes before extension; the containing encode operation still needs to connect matching source and reconstruction plane rents with ordinary `using` lifetimes. +- [~] A non-owning encoder-frame view now separates visible conversion regions from coded regions and performs complete left, top, right, bottom, and corner extension across each bordered plane. Current libaom uses 8-sample-aligned coded dimensions, a 32-sample-aligned luma stride with chroma stride derived from it, and a 64-pixel luma border for non-resized all-intra encoding. One operation-ready frame owner now rents the aligned Y, U, and V storage contiguously, exposes non-owning `Buffer2D` plane views, and returns the rent exactly once. A 4K 4:2:0 frame occupies about 13.0 MiB at 8-bit or 26.0 MiB at 10/12-bit; source and reconstruction therefore remain distinct frame owners rather than adding a full-frame copy. The corrected tests use this real ownership path and verify the exact 54 KiB 64x64 4:2:0 rent. The frame-encoder boundary converts packed pixels directly into the source owner before extension; the containing encode operation still needs to instantiate matching source and reconstruction owners with ordinary `using` lifetimes. - [~] Temporal delimiter, sequence header, frame header, and combined-frame tile-group writing exist locally. The remaining required metadata, padding, and encoder-wide syntax paths are not complete. - [ ] Implement superblock and partition analysis for every permitted block size and partition. - [ ] Implement intra mode search, chroma mode search, palette, filter intra, chroma-from-luma, and intra-block copy decisions. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs index 75c12fdacd..ba2cf49d3e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs @@ -172,6 +172,17 @@ internal readonly struct Av1EncoderFrame /// public int ChromaPositionY { get; } + /// + /// Calculates the coded luma dimensions used by the fixed all-intra frame layout. + /// + /// The visible luma width. + /// The visible luma height. + /// The visible dimensions rounded up to the coding alignment. + public static Size GetCodedSize(int width, int height) + => new( + Av1Math.AlignPowerOf2(width, CodedDimensionAlignmentLog2), + Av1Math.AlignPowerOf2(height, CodedDimensionAlignmentLog2)); + /// /// Calculates the physical dimensions required for an all-intra component plane. /// @@ -182,15 +193,14 @@ internal readonly struct Av1EncoderFrame /// The physical plane dimensions, including its complete border and row padding. public static Size GetPlaneBufferSize(int width, int height, int subsamplingX, int subsamplingY) { - int codedWidth = Av1Math.AlignPowerOf2(width, CodedDimensionAlignmentLog2); - int codedHeight = Av1Math.AlignPowerOf2(height, CodedDimensionAlignmentLog2); + Size codedSize = GetCodedSize(width, height); // libaom aligns the complete luma row before deriving a subsampled plane's stride. // Aligning chroma independently would produce a different physical layout for narrow or odd-sized frames. - int lumaStride = Av1Math.AlignPowerOf2(codedWidth + (2 * LumaBorder), LumaStrideAlignmentLog2); + int lumaStride = Av1Math.AlignPowerOf2(codedSize.Width + (2 * LumaBorder), LumaStrideAlignmentLog2); int planeStride = lumaStride >> subsamplingX; int planeBorderHeight = LumaBorder >> subsamplingY; - return new Size(planeStride, (codedHeight >> subsamplingY) + (2 * planeBorderHeight)); + return new Size(planeStride, (codedSize.Height >> subsamplingY) + (2 * planeBorderHeight)); } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs new file mode 100644 index 0000000000..9c429bd2a6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs @@ -0,0 +1,165 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers; +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Owns the aligned luma and chroma planes used by one AV1 encoder frame. +/// +/// The native unsigned sample storage type. +internal sealed class Av1EncoderFrameBuffer : IDisposable + where TSample : unmanaged +{ + /// + /// The complete frame owner, or after disposal. + /// + private IMemoryOwner? owner; + + /// + /// Initializes a new instance of the class. + /// + /// The configuration providing the frame allocator. + /// The visible luma width. + /// The visible luma height. + /// The native component precision. + /// The native luma and chroma sampling layout. + /// The horizontal chroma position in half-luma-sample units. + /// The vertical chroma position in half-luma-sample units. + public Av1EncoderFrameBuffer( + Configuration configuration, + int width, + int height, + int bitDepth, + Av1ColorFormat colorFormat, + int chromaPositionX, + int chromaPositionY) + { + int subsamplingX = colorFormat is Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422 ? 1 : 0; + int subsamplingY = colorFormat == Av1ColorFormat.Yuv420 ? 1 : 0; + Size codedSize = Av1EncoderFrame.GetCodedSize(width, height); + Size lumaSize = Av1EncoderFrame.GetPlaneBufferSize(width, height, 0, 0); + int lumaElementCount = checked(lumaSize.Width * lumaSize.Height); + Size chromaSize = colorFormat == Av1ColorFormat.Yuv400 + ? Size.Empty + : Av1EncoderFrame.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY); + + int chromaElementCount = checked(chromaSize.Width * chromaSize.Height); + int planeAlignment = Math.Max(32 / Unsafe.SizeOf(), 1); + int chromaBlueOffset = Align(lumaElementCount, planeAlignment); + int chromaRedOffset = Align(checked(chromaBlueOffset + chromaElementCount), planeAlignment); + int storageLength = colorFormat == Av1ColorFormat.Yuv400 + ? lumaElementCount + : checked(chromaRedOffset + chromaElementCount); + + // Libaom keeps the three component planes in one 32-byte-aligned frame allocation. The non-owning + // Buffer2D views preserve ImageSharp's row API without introducing separate plane rents or copies. + IMemoryOwner owner = configuration.MemoryAllocator.Allocate(storageLength); + Memory storage = owner.Memory; + Buffer2D luma = Buffer2D.WrapMemory( + storage[..lumaElementCount], + lumaSize.Width, + lumaSize.Height); + + this.Luma = luma; + + Buffer2DRegion lumaRegion = luma.GetRegion( + Av1EncoderFrame.LumaBorder, + Av1EncoderFrame.LumaBorder, + codedSize.Width, + codedSize.Height); + + Buffer2DRegion chromaBlueRegion = default; + Buffer2DRegion chromaRedRegion = default; + if (colorFormat != Av1ColorFormat.Yuv400) + { + Buffer2D chromaBlue = Buffer2D.WrapMemory( + storage.Slice(chromaBlueOffset, chromaElementCount), + chromaSize.Width, + chromaSize.Height); + + Buffer2D chromaRed = Buffer2D.WrapMemory( + storage.Slice(chromaRedOffset, chromaElementCount), + chromaSize.Width, + chromaSize.Height); + + this.ChromaBlue = chromaBlue; + this.ChromaRed = chromaRed; + + int chromaBorderX = Av1EncoderFrame.LumaBorder >> subsamplingX; + int chromaBorderY = Av1EncoderFrame.LumaBorder >> subsamplingY; + int codedChromaWidth = codedSize.Width >> subsamplingX; + int codedChromaHeight = codedSize.Height >> subsamplingY; + chromaBlueRegion = chromaBlue.GetRegion( + chromaBorderX, + chromaBorderY, + codedChromaWidth, + codedChromaHeight); + + chromaRedRegion = chromaRed.GetRegion( + chromaBorderX, + chromaBorderY, + codedChromaWidth, + codedChromaHeight); + } + + this.owner = owner; + this.Frame = new( + lumaRegion, + chromaBlueRegion, + chromaRedRegion, + width, + height, + bitDepth, + colorFormat, + chromaPositionX, + chromaPositionY); + } + + /// + /// Gets the non-owning coded frame view. + /// + public Av1EncoderFrame Frame { get; } + + /// + /// Gets the complete padded luma plane. + /// + public Buffer2D Luma { get; } + + /// + /// Gets the complete padded blue-difference chroma plane. + /// + public Buffer2D? ChromaBlue { get; } + + /// + /// Gets the complete padded red-difference chroma plane. + /// + public Buffer2D? ChromaRed { get; } + + /// + /// Releases the complete frame allocation. + /// + public void Dispose() + { + IMemoryOwner? ownedMemory = this.owner; + this.owner = null; + if (ownedMemory is null) + { + return; + } + + this.Luma.Dispose(); + this.ChromaBlue?.Dispose(); + this.ChromaRed?.Dispose(); + ownedMemory.Dispose(); + } + + /// + /// Aligns an element offset to the next component-plane boundary. + /// + private static int Align(int value, int alignment) + => checked((value + alignment - 1) & -alignment); +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs index b144c852af..54c0113783 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs @@ -244,6 +244,12 @@ public class Av1CoefficientsEntropyTests { Av1EncoderBlockStruct[] blocks = new Av1EncoderBlockStruct[2]; + // Exercise the inline-array accessors before measuring so one-time runtime generic initialization is + // excluded from the steady-state allocation contract used for every encoded block. + ref Av1EncoderBlockStruct warmupBlock = ref blocks[0]; + warmupBlock.PaletteSize[0] = 1; + warmupBlock.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = 1; + long before = GC.GetAllocatedBytesForCurrentThread(); ref Av1EncoderBlockStruct block = ref blocks[1]; block.PaletteSize[0] = 3; diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs index 875ba80e59..0915842cc3 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs @@ -6,6 +6,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Tests.Memory; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -16,10 +17,7 @@ public class Av1EncoderFrameTests { const int width = 4; const int height = 1; - const int codedWidth = 8; - const int codedHeight = 8; const int border = Av1EncoderFrame.LumaBorder; - MemoryAllocator allocator = Configuration.Default.MemoryAllocator; using Image image = new(width, height); image[0, 0] = new Rgba32(byte.MaxValue, 0, 0, 0); @@ -27,16 +25,21 @@ public class Av1EncoderFrameTests image[2, 0] = new Rgba32(0, 0, byte.MaxValue); image[3, 0] = new Rgba32(byte.MaxValue, byte.MaxValue, byte.MaxValue); - Size bufferSize = Av1EncoderFrame.GetPlaneBufferSize(width, height, 0, 0); - using Buffer2D luma = allocator.Allocate2D(bufferSize.Width, bufferSize.Height); - Buffer2DRegion lumaRegion = luma.GetRegion(border, border, codedWidth, codedHeight); - Av1EncoderFrame frame = new(lumaRegion, width, height, 8); + using Av1EncoderFrameBuffer frameBuffer = new( + Configuration.Default, + width, + height, + 8, + Av1ColorFormat.Yuv400, + 0, + 0); + ObuColorConfig colorConfig = CreateMonochromeColorConfig(Av1BitDepth.EightBit); - Av1FrameEncoder.PrepareSource(Configuration.Default, image.Frames.RootFrame, frame, colorConfig); + Av1FrameEncoder.PrepareSource(Configuration.Default, image.Frames.RootFrame, frameBuffer.Frame, colorConfig); byte[] expected = [76, 150, 29, 255]; - AssertReplicatedSingleRow(luma, border, expected); + AssertReplicatedSingleRow(frameBuffer.Luma, border, expected); } [Fact] @@ -44,10 +47,7 @@ public class Av1EncoderFrameTests { const int width = 4; const int height = 1; - const int codedWidth = 8; - const int codedHeight = 8; const int border = Av1EncoderFrame.LumaBorder; - MemoryAllocator allocator = Configuration.Default.MemoryAllocator; using Image image = new(width, height); image[0, 0] = new Rgba64(ushort.MaxValue, 0, 0, 0); @@ -55,16 +55,21 @@ public class Av1EncoderFrameTests image[2, 0] = new Rgba64(0, 0, ushort.MaxValue, ushort.MaxValue); image[3, 0] = new Rgba64(ushort.MaxValue, ushort.MaxValue, ushort.MaxValue, ushort.MaxValue); - Size bufferSize = Av1EncoderFrame.GetPlaneBufferSize(width, height, 0, 0); - using Buffer2D luma = allocator.Allocate2D(bufferSize.Width, bufferSize.Height); - Buffer2DRegion lumaRegion = luma.GetRegion(border, border, codedWidth, codedHeight); - Av1EncoderFrame frame = new(lumaRegion, width, height, 10); + using Av1EncoderFrameBuffer frameBuffer = new( + Configuration.Default, + width, + height, + 10, + Av1ColorFormat.Yuv400, + 0, + 0); + ObuColorConfig colorConfig = CreateMonochromeColorConfig(Av1BitDepth.TenBit); - Av1FrameEncoder.PrepareSource(Configuration.Default, image.Frames.RootFrame, frame, colorConfig); + Av1FrameEncoder.PrepareSource(Configuration.Default, image.Frames.RootFrame, frameBuffer.Frame, colorConfig); ushort[] expected = [306, 601, 117, 1023]; - AssertReplicatedSingleRow(luma, border, expected); + AssertReplicatedSingleRow(frameBuffer.Luma, border, expected); } [Fact] @@ -72,29 +77,11 @@ public class Av1EncoderFrameTests { const int visibleWidth = 5; const int visibleHeight = 3; - const int codedWidth = 8; - const int codedHeight = 8; const int lumaBorder = Av1EncoderFrame.LumaBorder; const int chromaBorder = lumaBorder / 2; - MemoryAllocator allocator = Configuration.Default.MemoryAllocator; - - Size lumaBufferSize = Av1EncoderFrame.GetPlaneBufferSize(visibleWidth, visibleHeight, 0, 0); - Size chromaBufferSize = Av1EncoderFrame.GetPlaneBufferSize(visibleWidth, visibleHeight, 1, 1); - using Buffer2D luma = allocator.Allocate2D(lumaBufferSize.Width, lumaBufferSize.Height); - using Buffer2D chromaBlue = allocator.Allocate2D(chromaBufferSize.Width, chromaBufferSize.Height); - using Buffer2D chromaRed = allocator.Allocate2D(chromaBufferSize.Width, chromaBufferSize.Height); - Buffer2DRegion lumaRegion = luma.GetRegion(lumaBorder, lumaBorder, codedWidth, codedHeight); - Buffer2DRegion chromaBlueRegion = chromaBlue.GetRegion(chromaBorder, chromaBorder, codedWidth / 2, codedHeight / 2); - Buffer2DRegion chromaRedRegion = chromaRed.GetRegion(chromaBorder, chromaBorder, codedWidth / 2, codedHeight / 2); - - FillVisible(luma, lumaBorder, lumaBorder, visibleWidth, visibleHeight, 10); - FillVisible(chromaBlue, chromaBorder, chromaBorder, (visibleWidth + 1) / 2, (visibleHeight + 1) / 2, 80); - FillVisible(chromaRed, chromaBorder, chromaBorder, (visibleWidth + 1) / 2, (visibleHeight + 1) / 2, 120); - Av1EncoderFrame frame = new( - lumaRegion, - chromaBlueRegion, - chromaRedRegion, + using Av1EncoderFrameBuffer frameBuffer = new( + Configuration.Default, visibleWidth, visibleHeight, 8, @@ -102,7 +89,15 @@ public class Av1EncoderFrameTests 1, 1); - frame.ExtendBorders(); + Buffer2D luma = frameBuffer.Luma; + Buffer2D chromaBlue = Assert.IsType>(frameBuffer.ChromaBlue); + Buffer2D chromaRed = Assert.IsType>(frameBuffer.ChromaRed); + + FillVisible(luma, lumaBorder, lumaBorder, visibleWidth, visibleHeight, 10); + FillVisible(chromaBlue, chromaBorder, chromaBorder, (visibleWidth + 1) / 2, (visibleHeight + 1) / 2, 80); + FillVisible(chromaRed, chromaBorder, chromaBorder, (visibleWidth + 1) / 2, (visibleHeight + 1) / 2, 120); + + frameBuffer.Frame.ExtendBorders(); AssertReplicatedPlane(luma, lumaBorder, lumaBorder, visibleWidth, visibleHeight, 10); AssertReplicatedPlane(chromaBlue, chromaBorder, chromaBorder, (visibleWidth + 1) / 2, (visibleHeight + 1) / 2, 80); @@ -128,6 +123,37 @@ public class Av1EncoderFrameTests Assert.Equal(new Size(expectedWidth, expectedHeight), actual); } + [Fact] + public void FrameBufferUsesOneExactSizeOwnerForAllPlanes() + { + TestMemoryAllocator allocator = new(); + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + + TestMemoryAllocator.AllocationRequest allocation; + using (Av1EncoderFrameBuffer frameBuffer = new( + configuration, + 64, + 64, + 8, + Av1ColorFormat.Yuv420, + 1, + 1)) + { + allocation = Assert.Single(allocator.AllocationLog); + Assert.Empty(allocator.ReturnLog); + Assert.Equal(typeof(byte), allocation.ElementType); + Assert.Equal(55_296, allocation.Length); + Assert.Single(frameBuffer.Luma.MemoryGroup); + Assert.Single(Assert.IsType>(frameBuffer.ChromaBlue).MemoryGroup); + Assert.Single(Assert.IsType>(frameBuffer.ChromaRed).MemoryGroup); + } + + TestMemoryAllocator.ReturnRequest returned = Assert.Single(allocator.ReturnLog); + Assert.Equal(allocation.AllocationId, returned.AllocationId); + } + private static ObuColorConfig CreateMonochromeColorConfig(Av1BitDepth bitDepth) => new() {