From 61b9cd4cb60f620b40281aeef7c8d6415457e108 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 24 Aug 2026 17:28:42 +1000 Subject: [PATCH] Support subsampled AV1 color output --- HEIF_IMPLEMENTATION_PLAN.md | 6 +- .../Formats/Heif/Av1/Av1FrameBuffer.cs | 14 +- .../Formats/Heif/Av1/Av1YuvConverter.cs | 191 ++++++++++++++---- .../ObuChromoSamplePosition.cs | 5 + .../Formats/Heif/Av1/Av1YuvConverterTests.cs | 113 ++++++++++- 5 files changed, 284 insertions(+), 45 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index e19bda196..88e3643da 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -56,7 +56,7 @@ This snapshot pins or classifies the available references and failures; it does | Managed implementation | Normative behavior | Reviewed implementation reference | Use | | --- | --- | --- | --- | -| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, and scalar row conversion | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826` | Scalar behavioral oracle for 8-bit YUV 4:4:4 full/limited-range conversion; later subsampling, high-bit-depth, and SIMD paths must match it. | +| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. | This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped. @@ -97,8 +97,8 @@ This assessment is based on the current source after the upstream ImageSharp mer - Decoder state is allocated or replaced at multiple points, making parsed tile state, reconstructed frame state, and ownership unclear. - The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements. - Loop restoration, filter intra prediction, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches. -- The active output path is limited to an 8-bit byte buffer and YUV 4:4:4. The generic frame buffer does not yet establish correct storage and indexing for 10/12-bit samples. -- `Av1YuvConverter` ignores most bitstream color information, uses fixed conversion constants, and allocates an intermediate `Image` before converting to the requested pixel type. +- The active reconstruction path remains limited to an 8-bit byte buffer. Output conversion now handles monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 planes, but the generic frame buffer does not yet establish correct storage and indexing for 10/12-bit samples. +- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8-bit output and uses one allocator-backed RGB row. High-bit-depth conversion, constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete. - The inverse-transform path allocates arrays in a per-transform hot path. - No usable end-to-end AV1 SIMD path was found. The most visible 4x4 forward-transform SIMD call is commented out, while the production prediction, transform, filter, and output paths are predominantly scalar. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs index 6bef7e899..d66b2c41e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs @@ -266,13 +266,23 @@ internal class Av1FrameBuffer : IDisposable case Av1Plane.U: Guard.NotNull(this.BufferCb); buffer = this.BufferCb; - region = new Rectangle(this.OriginX >> subX, this.OriginY >> subY, this.Width >> subX, this.Height >> subY); + region = new Rectangle( + this.OriginX >> subX, + this.OriginY >> subY, + Av1Math.DivideLog2Ceiling(this.Width, subX), + Av1Math.DivideLog2Ceiling(this.Height, subY)); + break; case Av1Plane.V: default: Guard.NotNull(this.BufferCr); buffer = this.BufferCr; - region = new Rectangle(this.OriginX >> subX, this.OriginY >> subY, this.Width >> subX, this.Height >> subY); + region = new Rectangle( + this.OriginX >> subX, + this.OriginY >> subY, + Av1Math.DivideLog2Ceiling(this.Width, subX), + Av1Math.DivideLog2Ceiling(this.Height, subY)); + break; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs index e004243a9..6a44b50e7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs @@ -26,7 +26,7 @@ internal static class Av1YuvConverter } /// - /// Converts the reconstructed 8-bit YUV 4:4:4 planes to packed pixels. + /// Converts the reconstructed 8-bit YUV planes to packed pixels. /// /// The destination pixel type. /// The configuration used for allocation and pixel conversion. @@ -46,18 +46,26 @@ internal static class Av1YuvConverter out float chromaScale); Buffer2DRegion yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0); - Buffer2DRegion uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 0, 0); - Buffer2DRegion vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 0, 0); + bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400; + int subX = frameBuffer.ColorConfig.SubSamplingX ? 1 : 0; + int subY = frameBuffer.ColorConfig.SubSamplingY ? 1 : 0; + Buffer2DRegion uPlane = isMonochrome ? default : frameBuffer.DeriveBlockPointer(Av1Plane.U, subX, subY); + Buffer2DRegion vPlane = isMonochrome ? default : frameBuffer.DeriveBlockPointer(Av1Plane.V, subX, subY); using IMemoryOwner rowOwner = configuration.MemoryAllocator.Allocate(image.Width); Span rgbRow = rowOwner.GetSpan()[..image.Width]; for (int y = 0; y < image.Height; y++) { - ConvertYuv444ToRgbRow( + ConvertYuvToRgbRow( yPlane.DangerousGetRowSpan(y), - uPlane.DangerousGetRowSpan(y), - vPlane.DangerousGetRowSpan(y), + uPlane, + vPlane, + y, rgbRow, + isMonochrome, + subX, + subY, + frameBuffer.ColorConfig.ChromaSamplePosition, mode, kr, kg, @@ -93,6 +101,11 @@ internal static class Av1YuvConverter out float lumaScale, out float chromaScale); + if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444) + { + throw new NotSupportedException("Only AV1 YUV 4:4:4 encoding color conversion is currently supported."); + } + Buffer2DRegion yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0); Buffer2DRegion uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 0, 0); Buffer2DRegion vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 0, 0); @@ -147,11 +160,6 @@ internal static class Av1YuvConverter throw new NotSupportedException("Only 8-bit AV1 color conversion is currently supported."); } - if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444) - { - throw new NotSupportedException("Only AV1 YUV 4:4:4 color conversion is currently supported."); - } - mode = ConversionMode.Coefficients; kr = 0F; kb = 0F; @@ -193,8 +201,19 @@ internal static class Av1YuvConverter } kg = 1F - kr - kb; + bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400; bool isFullRange = frameBuffer.ColorConfig.ColorRange; - if (mode == ConversionMode.YCgCo && !isFullRange) + if (mode == ConversionMode.Identity && !isMonochrome && frameBuffer.ColorFormat != Av1ColorFormat.Yuv444) + { + throw new InvalidImageContentException("AV1 identity matrix coefficients require YUV 4:4:4 sampling."); + } + + if (frameBuffer.ColorConfig.ChromaSamplePosition == ObuChromoSamplePosition.Reserved) + { + throw new InvalidImageContentException("The reserved AV1 chroma sample position is invalid."); + } + + if (mode == ConversionMode.YCgCo && !isMonochrome && !isFullRange) { throw new NotSupportedException("Limited-range AV1 YCgCo color conversion is not currently supported."); } @@ -205,12 +224,17 @@ internal static class Av1YuvConverter } /// - /// Converts one YUV 4:4:4 row to packed RGB using the resolved H.273 conversion state. + /// Converts one YUV row to packed RGB using the resolved H.273 conversion state. /// /// The luma samples. - /// The blue-difference chroma samples. - /// The red-difference chroma samples. + /// The blue-difference chroma plane. + /// The red-difference chroma plane. + /// The luma row coordinate. /// The destination RGB pixels. + /// Whether the frame contains only luma samples. + /// The horizontal chroma subsampling shift. + /// The vertical chroma subsampling shift. + /// The spatial position of subsampled chroma. /// The conversion mode. /// The red luma coefficient. /// The green luma coefficient. @@ -218,11 +242,16 @@ internal static class Av1YuvConverter /// The encoded luma bias. /// The encoded luma range. /// The encoded chroma range. - private static void ConvertYuv444ToRgbRow( + private static void ConvertYuvToRgbRow( ReadOnlySpan ySource, - ReadOnlySpan uSource, - ReadOnlySpan vSource, + in Buffer2DRegion uPlane, + in Buffer2DRegion vPlane, + int yCoordinate, Span destination, + bool isMonochrome, + int subX, + int subY, + ObuChromoSamplePosition chromaSamplePosition, ConversionMode mode, float kr, float kg, @@ -234,31 +263,43 @@ internal static class Av1YuvConverter for (int x = 0; x < destination.Length; x++) { float y = (ySource[x] - lumaBias) / lumaScale; - float cb = (uSource[x] - ChromaBias) / chromaScale; - float cr = (vSource[x] - ChromaBias) / chromaScale; float r; float g; float b; - switch (mode) + if (isMonochrome) { - case ConversionMode.Identity: - // H.273 identity coding stores the nonlinear G, B, and R signals in Y, U, and V order. - r = (vSource[x] - lumaBias) / lumaScale; - g = y; - b = (uSource[x] - lumaBias) / lumaScale; - break; - case ConversionMode.YCgCo: - float temporary = y - cb; - r = temporary + cr; - g = y + cb; - b = temporary - cr; - break; - default: - r = y + (2F * (1F - kr) * cr); - g = y - (2F * ((kr * (1F - kr) * cr) + (kb * (1F - kb) * cb)) / kg); - b = y + (2F * (1F - kb) * cb); - break; + r = y; + g = y; + b = y; + } + else + { + float u = SampleChroma(uPlane, x, yCoordinate, subX, subY, chromaSamplePosition); + float v = SampleChroma(vPlane, x, yCoordinate, subX, subY, chromaSamplePosition); + float cb = (u - ChromaBias) / chromaScale; + float cr = (v - ChromaBias) / chromaScale; + + switch (mode) + { + case ConversionMode.Identity: + // H.273 identity coding stores the nonlinear G, B, and R signals in Y, U, and V order. + r = (v - lumaBias) / lumaScale; + g = y; + b = (u - lumaBias) / lumaScale; + break; + case ConversionMode.YCgCo: + float temporary = y - cb; + r = temporary + cr; + g = y + cb; + b = temporary - cr; + break; + default: + r = y + (2F * (1F - kr) * cr); + g = y - (2F * ((kr * (1F - kr) * cr) + (kb * (1F - kb) * cb)) / kg); + b = y + (2F * (1F - kb) * cb); + break; + } } destination[x] = new Rgb24( @@ -268,6 +309,80 @@ internal static class Av1YuvConverter } } + /// + /// Bilinearly reconstructs a chroma sample at a luma coordinate. + /// + /// The subsampled chroma plane. + /// The luma column coordinate. + /// The luma row coordinate. + /// The horizontal chroma subsampling shift. + /// The vertical chroma subsampling shift. + /// The spatial position of subsampled chroma. + /// The reconstructed encoded chroma sample. + private static float SampleChroma( + in Buffer2DRegion plane, + int x, + int y, + int subX, + int subY, + ObuChromoSamplePosition chromaSamplePosition) + { + // Unknown 4:2:0 and all 4:2:2 input use the centered convention employed by libavif. + bool isCenteredX = subX != 0 && (subY == 0 || chromaSamplePosition == ObuChromoSamplePosition.Unknown); + bool isCenteredY = subY != 0 && chromaSamplePosition != ObuChromoSamplePosition.Colocated; + GetChromaCoordinates(x, subX, isCenteredX, plane.Width - 1, out int x0, out int x1, out int x1Weight); + GetChromaCoordinates(y, subY, isCenteredY, plane.Height - 1, out int y0, out int y1, out int y1Weight); + + ReadOnlySpan row0 = plane.DangerousGetRowSpan(y0); + ReadOnlySpan row1 = plane.DangerousGetRowSpan(y1); + int top = (row0[x0] * (4 - x1Weight)) + (row0[x1] * x1Weight); + int bottom = (row1[x0] * (4 - x1Weight)) + (row1[x1] * x1Weight); + + return ((top * (4 - y1Weight)) + (bottom * y1Weight)) / 16F; + } + + /// + /// Resolves the two chroma samples and quarter-sample weight surrounding a luma coordinate. + /// + /// The luma coordinate. + /// The chroma subsampling shift. + /// Whether chroma lies between neighboring luma samples. + /// The last available chroma coordinate. + /// The lower chroma coordinate. + /// The upper chroma coordinate. + /// The upper-coordinate weight with a denominator of four. + private static void GetChromaCoordinates( + int coordinate, + int subsampling, + bool isCentered, + int maximum, + out int lower, + out int upper, + out int upperWeight) + { + if (subsampling == 0) + { + lower = coordinate; + upper = coordinate; + upperWeight = 0; + return; + } + + int sample = coordinate >> 1; + bool isOdd = (coordinate & 1) != 0; + if (isCentered) + { + lower = isOdd ? sample : Math.Max(sample - 1, 0); + upper = isOdd ? Math.Min(sample + 1, maximum) : sample; + upperWeight = isOdd ? 1 : 3; + return; + } + + lower = sample; + upper = isOdd ? Math.Min(sample + 1, maximum) : sample; + upperWeight = isOdd ? 2 : 0; + } + /// /// Converts one packed RGB row to YUV 4:4:4 using the resolved H.273 conversion state. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs index bd71ea433..f08d2ab0b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs @@ -19,4 +19,9 @@ internal enum ObuChromoSamplePosition : byte /// Co-located with luma(0, 0) sample /// Colocated = 2, + + /// + /// Reserved and invalid for AV1 content. + /// + Reserved = 3, } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs index 960d8102e..70badeec5 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs @@ -91,6 +91,110 @@ public class Av1YuvConverterTests Assert.Equal(b, actual.B, 1d); } + [Fact] + public void Yuv400ToRgbExpandsLimitedRangeLuma() + { + // Assign + using Image image = new(2, 1); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + 2, + 1, + false, + ObuMatrixCoefficients.Identity, + Av1ColorFormat.Yuv400); + + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + Span yRow = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0).DangerousGetRowSpan(0); + yRow[0] = 16; + yRow[1] = 235; + + // Act + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame); + + // Assert + Span actual = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0); + Assert.Equal(new Rgb24(0, 0, 0), actual[0]); + Assert.Equal(new Rgb24(255, 255, 255), actual[1]); + } + + [Fact] + public void Yuv422ToRgbBilinearlyUpsamplesCenteredChroma() + { + // Assign + using Image image = new(4, 1); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(4, 1, colorFormat: Av1ColorFormat.Yuv422); + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv422, false); + frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0).DangerousGetRowSpan(0).Fill(128); + Span uRow = frameBuffer.DeriveBlockPointer(Av1Plane.U, 1, 0).DangerousGetRowSpan(0); + uRow[0] = 128; + uRow[1] = 192; + frameBuffer.DeriveBlockPointer(Av1Plane.V, 1, 0).DangerousGetRowSpan(0).Fill(128); + + // Act + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame); + + // Assert + Span actual = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0); + Assert.Equal(new Rgb24(128, 128, 128), actual[0]); + Assert.Equal(new Rgb24(128, 125, 158), actual[1]); + Assert.Equal(new Rgb24(128, 119, 217), actual[2]); + Assert.Equal(new Rgb24(128, 116, 247), actual[3]); + } + + [Theory] + [InlineData(ObuChromoSamplePosition.Unknown, 158, 98)] + [InlineData(ObuChromoSamplePosition.Vertical, 187, 98)] + [InlineData(ObuChromoSamplePosition.Colocated, 187, 69)] + public void Yuv420ToRgbUsesChromaSamplePosition(int chromaSamplePosition, byte expectedTopBlue, byte expectedLeftBlue) + { + // Assign + using Image image = new(4, 4); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + 4, + 4, + colorFormat: Av1ColorFormat.Yuv420, + chromaSamplePosition: (ObuChromoSamplePosition)chromaSamplePosition); + + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false); + Buffer2DRegion yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0); + Buffer2DRegion uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 1, 1); + Buffer2DRegion vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 1, 1); + for (int y = 0; y < yPlane.Height; y++) + { + yPlane.DangerousGetRowSpan(y).Fill(128); + } + + uPlane.DangerousGetRowSpan(0)[0] = 128; + uPlane.DangerousGetRowSpan(0)[1] = 192; + uPlane.DangerousGetRowSpan(1)[0] = 64; + uPlane.DangerousGetRowSpan(1)[1] = 255; + vPlane.DangerousGetRowSpan(0).Fill(128); + vPlane.DangerousGetRowSpan(1).Fill(128); + + // Act + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame); + + // Assert + Assert.Equal(expectedTopBlue, image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0)[1].B); + Assert.Equal(expectedLeftBlue, image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(1)[0].B); + } + + [Fact] + public void Yuv420UsesCeilingChromaPlaneDimensions() + { + // Assign + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(3, 3, colorFormat: Av1ColorFormat.Yuv420); + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false); + + // Act + Buffer2DRegion uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 1, 1); + Buffer2DRegion vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 1, 1); + + // Assert + Assert.Equal(new Size(2, 2), uPlane.Size); + Assert.Equal(new Size(2, 2), vPlane.Size); + } + [Fact] public void RgbToYuvCompareToReferenceRandomPixels() { @@ -252,17 +356,22 @@ public class Av1YuvConverterTests int width, int height, bool fullRange = true, - ObuMatrixCoefficients matrixCoefficients = ObuMatrixCoefficients.Bt709) + ObuMatrixCoefficients matrixCoefficients = ObuMatrixCoefficients.Bt709, + Av1ColorFormat colorFormat = Av1ColorFormat.Yuv444, + ObuChromoSamplePosition chromaSamplePosition = ObuChromoSamplePosition.Unknown) => new() { MaxFrameWidth = width, MaxFrameHeight = height, ColorConfig = new ObuColorConfig { - IsMonochrome = false, + IsMonochrome = colorFormat == Av1ColorFormat.Yuv400, BitDepth = Av1BitDepth.EightBit, MatrixCoefficients = matrixCoefficients, ColorRange = fullRange, + SubSamplingX = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422, + SubSamplingY = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420, + ChromaSamplePosition = chromaSamplePosition, }, }; }