From ac0fd9b4a67e5f4a2b76e91fa66b64f1fa2f0eeb Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 2 Sep 2026 10:59:31 +1000 Subject: [PATCH] Prepare bordered AV1 encoder source frames --- HEIF_IMPLEMENTATION_PLAN.md | 4 +- .../Formats/Heif/Av1/Color/Av1YuvConverter.cs | 30 ++++--- .../Heif/Av1/Pipeline/Av1FrameEncoder.cs | 74 +++++++++++---- .../Formats/Heif/Av1/Av1EncoderFrameTests.cs | 90 +++++++++++++++++++ 4 files changed, 167 insertions(+), 31 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index b925710b33..0de3d8891b 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -810,10 +810,10 @@ Encoder verification contract: ### 6. Build the complete AV1 frame encoder -- [~] SIMD-first RGB-to-native-plane conversion exists locally. +- [~] SIMD-first RGB-to-native-plane conversion now feeds eight-bit and high-bit-depth bordered AV1 source frames directly, preserving ImageSharp's arbitrary packed-pixel input contract without an intermediate full-frame native-plane copy. - [~] Forward transform families and transform workspace exist locally. - [~] Symbol writer, coefficient writer, and tile writer fragments exist locally. -- [~] A non-owning encoder-frame view now separates visible conversion regions from coded regions and performs complete left, top, right, bottom, and corner extension across each bordered plane. Current libaom uses 8-sample-aligned coded dimensions, a 32-sample-aligned luma stride with chroma stride derived from it, and a 64-pixel luma border for non-resized all-intra encoding. Allocator-backed luma and 4:2:0 chroma extension passed direct net11 VSTest; the containing encode operation still needs to connect matching plane rents with ordinary `using` lifetimes. +- [~] A non-owning encoder-frame view now separates visible conversion regions from coded regions and performs complete left, top, right, bottom, and corner extension across each bordered plane. Current libaom uses 8-sample-aligned coded dimensions, a 32-sample-aligned luma stride with chroma stride derived from it, and a 64-pixel luma border for non-resized all-intra encoding. The frame-encoder boundary now converts packed pixels directly into those final source planes before extension; the containing encode operation still needs to connect matching source and reconstruction plane rents with ordinary `using` lifetimes. - [~] Temporal delimiter, sequence header, frame header, and combined-frame tile-group writing exist locally. The remaining required metadata, padding, and encoder-wide syntax paths are not complete. - [ ] Implement superblock and partition analysis for every permitted block size and partition. - [ ] Implement intra mode search, chroma mode search, palette, filter intra, chroma-from-luma, and intra-block copy decisions. diff --git a/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs index f6b0c50eb2..18040bb514 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs @@ -24,7 +24,7 @@ internal static class Av1YuvConverter public static void ConvertToRgb(Configuration configuration, Av1FrameBuffer frameBuffer, ImageFrame image) where TPixel : unmanaged, IPixel { - HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer, out HeifColorConversionMode mode); + HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out HeifColorConversionMode mode); if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { Av1PlanarSampleBuffer buffer = new(frameBuffer); @@ -111,7 +111,7 @@ internal static class Av1YuvConverter bool premultiplied) where TPixel : unmanaged, IPixel { - HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer, out _); + HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out _); Rectangle sourceRectangle = new(0, 0, frameBuffer.Width, frameBuffer.Height); if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { @@ -151,7 +151,7 @@ internal static class Av1YuvConverter public static void ConvertFromRgb(Configuration configuration, ImageFrame image, Av1FrameBuffer frameBuffer) where TPixel : unmanaged, IPixel { - HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer, out HeifColorConversionMode mode); + HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out HeifColorConversionMode mode); if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { Av1PlanarSampleBuffer buffer = new(frameBuffer); @@ -177,27 +177,29 @@ internal static class Av1YuvConverter /// /// Resolves the H.273 conversion mode, matrix coefficients, and sample range for a frame. /// - /// The AV1 frame containing the signaled color configuration. + /// The signaled AV1 color configuration. /// The resolved conversion mode. /// The resolved conversion parameters. - private static HeifColorConversionParameters GetConversionParameters(Av1FrameBuffer frameBuffer, out HeifColorConversionMode mode) + public static HeifColorConversionParameters GetConversionParameters( + ObuColorConfig colorConfig, + out HeifColorConversionMode mode) { - if (frameBuffer.ColorConfig.ChromaSamplePosition == ObuChromoSamplePosition.Reserved) + if (colorConfig.ChromaSamplePosition == ObuChromoSamplePosition.Reserved) { throw new InvalidImageContentException("The reserved AV1 chroma sample position is invalid."); } - bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400; + bool isMonochrome = colorConfig.IsMonochrome; return HeifColorConversionParameters.Create( - (CicpColorPrimaries)(byte)frameBuffer.ColorConfig.ColorPrimaries, - (CicpTransferCharacteristics)(byte)frameBuffer.ColorConfig.TransferCharacteristics, - (CicpMatrixCoefficients)(byte)frameBuffer.ColorConfig.MatrixCoefficients, - frameBuffer.ColorConfig.ColorRange, - frameBuffer.BitDepth.GetBitCount(), - frameBuffer.BitDepth.GetBitCount(), + (CicpColorPrimaries)(byte)colorConfig.ColorPrimaries, + (CicpTransferCharacteristics)(byte)colorConfig.TransferCharacteristics, + (CicpMatrixCoefficients)(byte)colorConfig.MatrixCoefficients, + colorConfig.ColorRange, + colorConfig.BitDepth.GetBitCount(), + colorConfig.BitDepth.GetBitCount(), isMonochrome, - frameBuffer.ColorFormat == Av1ColorFormat.Yuv444, + colorConfig.GetColorFormat() == Av1ColorFormat.Yuv444, out mode); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs index 105083eeca..2f769c70a0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs @@ -1,35 +1,79 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Color; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Components; +using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// -/// Defines the work-in-progress AV1 still-image frame-encoding boundary. +/// Performs operation-scoped AV1 still-image frame encoding. /// -internal class Av1FrameEncoder +internal static class Av1FrameEncoder { /// - /// The source plane samples supplied for frame encoding. + /// Converts packed pixels directly into an eight-bit bordered AV1 source frame. /// - private readonly Av1FrameBuffer frameBuffer; + /// The packed source pixel type. + /// The configuration used for row-buffer allocation and pixel conversion. + /// The packed source frame. + /// The operation-owned AV1 source planes. + /// The color configuration written to the AV1 sequence header. + public static void PrepareSource( + Configuration configuration, + ImageFrame image, + Av1EncoderFrame source, + ObuColorConfig colorConfig) + where TPixel : unmanaged, IPixel + => PrepareSource(configuration, image, source, colorConfig); /// - /// Initializes a new instance of the class. + /// Converts packed pixels directly into a high-bit-depth bordered AV1 source frame. /// - /// The source frame samples to encode. - public Av1FrameEncoder(Av1FrameBuffer frameBuffer) - { - this.frameBuffer = frameBuffer; - } + /// The packed source pixel type. + /// The configuration used for row-buffer allocation and pixel conversion. + /// The packed source frame. + /// The operation-owned AV1 source planes. + /// The color configuration written to the AV1 sequence header. + public static void PrepareSource( + Configuration configuration, + ImageFrame image, + Av1EncoderFrame source, + ObuColorConfig colorConfig) + where TPixel : unmanaged, IPixel + => PrepareSource(configuration, image, source, colorConfig); /// - /// Represents the not-yet-implemented entry point for encoding one AV1 still-image frame. + /// Converts packed pixels into native component planes and initializes every coded and physical edge sample. /// - public static void Encode() + private static void PrepareSource( + Configuration configuration, + ImageFrame image, + Av1EncoderFrame source, + ObuColorConfig colorConfig) + where TPixel : unmanaged, IPixel + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter { - // Still-image encoding needs the normative analysis, transform, quantization, entropy, and packetization stages, - // but it does not require the encoder's application-level worker graph or video-sequence process orchestration. + HeifColorConversionParameters parameters = Av1YuvConverter.GetConversionParameters( + colorConfig, + out HeifColorConversionMode mode); + + // Conversion writes into the final bordered analysis planes. The later coding stages therefore consume the + // native source directly without a second full-frame copy from an intermediate component buffer. + HeifPlanarColorConverter.ConvertFromRgb< + TPixel, + Av1EncoderFrame.PlanarView, + TSample, + TStorer>( + configuration, + image, + source.View, + in parameters, + mode); + + source.ExtendBorders(); } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs index fc8c51a5f5..875ba80e59 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs @@ -2,13 +2,71 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; public class Av1EncoderFrameTests { + [Fact] + public void PrepareSourceConvertsRgba32DirectlyIntoBorderedEightBitPlane() + { + const int width = 4; + const int height = 1; + const int codedWidth = 8; + const int codedHeight = 8; + const int border = Av1EncoderFrame.LumaBorder; + MemoryAllocator allocator = Configuration.Default.MemoryAllocator; + + using Image image = new(width, height); + image[0, 0] = new Rgba32(byte.MaxValue, 0, 0, 0); + image[1, 0] = new Rgba32(0, byte.MaxValue, 0); + image[2, 0] = new Rgba32(0, 0, byte.MaxValue); + image[3, 0] = new Rgba32(byte.MaxValue, byte.MaxValue, byte.MaxValue); + + Size bufferSize = Av1EncoderFrame.GetPlaneBufferSize(width, height, 0, 0); + using Buffer2D luma = allocator.Allocate2D(bufferSize.Width, bufferSize.Height); + Buffer2DRegion lumaRegion = luma.GetRegion(border, border, codedWidth, codedHeight); + Av1EncoderFrame frame = new(lumaRegion, width, height, 8); + ObuColorConfig colorConfig = CreateMonochromeColorConfig(Av1BitDepth.EightBit); + + Av1FrameEncoder.PrepareSource(Configuration.Default, image.Frames.RootFrame, frame, colorConfig); + + byte[] expected = [76, 150, 29, 255]; + AssertReplicatedSingleRow(luma, border, expected); + } + + [Fact] + public void PrepareSourcePreservesHighBitDepthPrecision() + { + const int width = 4; + const int height = 1; + const int codedWidth = 8; + const int codedHeight = 8; + const int border = Av1EncoderFrame.LumaBorder; + MemoryAllocator allocator = Configuration.Default.MemoryAllocator; + + using Image image = new(width, height); + image[0, 0] = new Rgba64(ushort.MaxValue, 0, 0, 0); + image[1, 0] = new Rgba64(0, ushort.MaxValue, 0, ushort.MaxValue); + image[2, 0] = new Rgba64(0, 0, ushort.MaxValue, ushort.MaxValue); + image[3, 0] = new Rgba64(ushort.MaxValue, ushort.MaxValue, ushort.MaxValue, ushort.MaxValue); + + Size bufferSize = Av1EncoderFrame.GetPlaneBufferSize(width, height, 0, 0); + using Buffer2D luma = allocator.Allocate2D(bufferSize.Width, bufferSize.Height); + Buffer2DRegion lumaRegion = luma.GetRegion(border, border, codedWidth, codedHeight); + Av1EncoderFrame frame = new(lumaRegion, width, height, 10); + ObuColorConfig colorConfig = CreateMonochromeColorConfig(Av1BitDepth.TenBit); + + Av1FrameEncoder.PrepareSource(Configuration.Default, image.Frames.RootFrame, frame, colorConfig); + + ushort[] expected = [306, 601, 117, 1023]; + AssertReplicatedSingleRow(luma, border, expected); + } + [Fact] public void ExtendBordersReplicatesEveryPhysicalPlaneEdge() { @@ -70,6 +128,21 @@ public class Av1EncoderFrameTests Assert.Equal(new Size(expectedWidth, expectedHeight), actual); } + private static ObuColorConfig CreateMonochromeColorConfig(Av1BitDepth bitDepth) + => new() + { + IsColorDescriptionPresent = true, + IsMonochrome = true, + ColorPrimaries = ObuColorPrimaries.Bt601, + TransferCharacteristics = ObuTransferCharacteristics.Bt601, + MatrixCoefficients = ObuMatrixCoefficients.Bt601, + ColorRange = true, + SubSamplingX = true, + SubSamplingY = true, + ChromaSamplePosition = ObuChromoSamplePosition.Unknown, + BitDepth = bitDepth + }; + private static void FillVisible(Buffer2D plane, int originX, int originY, int width, int height, int seed) { for (int y = 0; y < height; y++) @@ -95,4 +168,21 @@ public class Av1EncoderFrameTests } } } + + private static void AssertReplicatedSingleRow( + Buffer2D plane, + int originX, + ReadOnlySpan expected) + where TSample : unmanaged, IEquatable + { + for (int y = 0; y < plane.Height; y++) + { + ReadOnlySpan row = plane.DangerousGetRowSpan(y); + for (int x = 0; x < row.Length; x++) + { + int sourceX = Math.Clamp(x - originX, 0, expected.Length - 1); + Assert.Equal(expected[sourceX], row[x]); + } + } + } }