Browse Source

Support subsampled AV1 color output

pull/2633/head
James Jackson-South 1 week ago
parent
commit
61b9cd4cb6
  1. 6
      HEIF_IMPLEMENTATION_PLAN.md
  2. 14
      src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs
  3. 191
      src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs
  4. 5
      src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs
  5. 113
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs

6
HEIF_IMPLEMENTATION_PLAN.md

@ -56,7 +56,7 @@ This snapshot pins or classifies the available references and failures; it does
| Managed implementation | Normative behavior | Reviewed implementation reference | Use |
| --- | --- | --- | --- |
| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, and scalar row conversion | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826` | Scalar behavioral oracle for 8-bit YUV 4:4:4 full/limited-range conversion; later subsampling, high-bit-depth, and SIMD paths must match it. |
| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. |
This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped.
@ -97,8 +97,8 @@ This assessment is based on the current source after the upstream ImageSharp mer
- Decoder state is allocated or replaced at multiple points, making parsed tile state, reconstructed frame state, and ownership unclear.
- The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements.
- Loop restoration, filter intra prediction, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches.
- The active output path is limited to an 8-bit byte buffer and YUV 4:4:4. The generic frame buffer does not yet establish correct storage and indexing for 10/12-bit samples.
- `Av1YuvConverter` ignores most bitstream color information, uses fixed conversion constants, and allocates an intermediate `Image<Rgb24>` before converting to the requested pixel type.
- The active reconstruction path remains limited to an 8-bit byte buffer. Output conversion now handles monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 planes, but the generic frame buffer does not yet establish correct storage and indexing for 10/12-bit samples.
- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8-bit output and uses one allocator-backed RGB row. High-bit-depth conversion, constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete.
- The inverse-transform path allocates arrays in a per-transform hot path.
- No usable end-to-end AV1 SIMD path was found. The most visible 4x4 forward-transform SIMD call is commented out, while the production prediction, transform, filter, and output paths are predominantly scalar.

14
src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs

@ -266,13 +266,23 @@ internal class Av1FrameBuffer<T> : IDisposable
case Av1Plane.U:
Guard.NotNull(this.BufferCb);
buffer = this.BufferCb;
region = new Rectangle(this.OriginX >> subX, this.OriginY >> subY, this.Width >> subX, this.Height >> subY);
region = new Rectangle(
this.OriginX >> subX,
this.OriginY >> subY,
Av1Math.DivideLog2Ceiling(this.Width, subX),
Av1Math.DivideLog2Ceiling(this.Height, subY));
break;
case Av1Plane.V:
default:
Guard.NotNull(this.BufferCr);
buffer = this.BufferCr;
region = new Rectangle(this.OriginX >> subX, this.OriginY >> subY, this.Width >> subX, this.Height >> subY);
region = new Rectangle(
this.OriginX >> subX,
this.OriginY >> subY,
Av1Math.DivideLog2Ceiling(this.Width, subX),
Av1Math.DivideLog2Ceiling(this.Height, subY));
break;
}

191
src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs

@ -26,7 +26,7 @@ internal static class Av1YuvConverter
}
/// <summary>
/// Converts the reconstructed 8-bit YUV 4:4:4 planes to packed pixels.
/// Converts the reconstructed 8-bit YUV planes to packed pixels.
/// </summary>
/// <typeparam name="TPixel">The destination pixel type.</typeparam>
/// <param name="configuration">The configuration used for allocation and pixel conversion.</param>
@ -46,18 +46,26 @@ internal static class Av1YuvConverter
out float chromaScale);
Buffer2DRegion<byte> yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0);
Buffer2DRegion<byte> uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 0, 0);
Buffer2DRegion<byte> vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 0, 0);
bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400;
int subX = frameBuffer.ColorConfig.SubSamplingX ? 1 : 0;
int subY = frameBuffer.ColorConfig.SubSamplingY ? 1 : 0;
Buffer2DRegion<byte> uPlane = isMonochrome ? default : frameBuffer.DeriveBlockPointer(Av1Plane.U, subX, subY);
Buffer2DRegion<byte> vPlane = isMonochrome ? default : frameBuffer.DeriveBlockPointer(Av1Plane.V, subX, subY);
using IMemoryOwner<Rgb24> rowOwner = configuration.MemoryAllocator.Allocate<Rgb24>(image.Width);
Span<Rgb24> rgbRow = rowOwner.GetSpan()[..image.Width];
for (int y = 0; y < image.Height; y++)
{
ConvertYuv444ToRgbRow(
ConvertYuvToRgbRow(
yPlane.DangerousGetRowSpan(y),
uPlane.DangerousGetRowSpan(y),
vPlane.DangerousGetRowSpan(y),
uPlane,
vPlane,
y,
rgbRow,
isMonochrome,
subX,
subY,
frameBuffer.ColorConfig.ChromaSamplePosition,
mode,
kr,
kg,
@ -93,6 +101,11 @@ internal static class Av1YuvConverter
out float lumaScale,
out float chromaScale);
if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444)
{
throw new NotSupportedException("Only AV1 YUV 4:4:4 encoding color conversion is currently supported.");
}
Buffer2DRegion<byte> yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0);
Buffer2DRegion<byte> uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 0, 0);
Buffer2DRegion<byte> vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 0, 0);
@ -147,11 +160,6 @@ internal static class Av1YuvConverter
throw new NotSupportedException("Only 8-bit AV1 color conversion is currently supported.");
}
if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444)
{
throw new NotSupportedException("Only AV1 YUV 4:4:4 color conversion is currently supported.");
}
mode = ConversionMode.Coefficients;
kr = 0F;
kb = 0F;
@ -193,8 +201,19 @@ internal static class Av1YuvConverter
}
kg = 1F - kr - kb;
bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400;
bool isFullRange = frameBuffer.ColorConfig.ColorRange;
if (mode == ConversionMode.YCgCo && !isFullRange)
if (mode == ConversionMode.Identity && !isMonochrome && frameBuffer.ColorFormat != Av1ColorFormat.Yuv444)
{
throw new InvalidImageContentException("AV1 identity matrix coefficients require YUV 4:4:4 sampling.");
}
if (frameBuffer.ColorConfig.ChromaSamplePosition == ObuChromoSamplePosition.Reserved)
{
throw new InvalidImageContentException("The reserved AV1 chroma sample position is invalid.");
}
if (mode == ConversionMode.YCgCo && !isMonochrome && !isFullRange)
{
throw new NotSupportedException("Limited-range AV1 YCgCo color conversion is not currently supported.");
}
@ -205,12 +224,17 @@ internal static class Av1YuvConverter
}
/// <summary>
/// Converts one YUV 4:4:4 row to packed RGB using the resolved H.273 conversion state.
/// Converts one YUV row to packed RGB using the resolved H.273 conversion state.
/// </summary>
/// <param name="ySource">The luma samples.</param>
/// <param name="uSource">The blue-difference chroma samples.</param>
/// <param name="vSource">The red-difference chroma samples.</param>
/// <param name="uPlane">The blue-difference chroma plane.</param>
/// <param name="vPlane">The red-difference chroma plane.</param>
/// <param name="yCoordinate">The luma row coordinate.</param>
/// <param name="destination">The destination RGB pixels.</param>
/// <param name="isMonochrome">Whether the frame contains only luma samples.</param>
/// <param name="subX">The horizontal chroma subsampling shift.</param>
/// <param name="subY">The vertical chroma subsampling shift.</param>
/// <param name="chromaSamplePosition">The spatial position of subsampled chroma.</param>
/// <param name="mode">The conversion mode.</param>
/// <param name="kr">The red luma coefficient.</param>
/// <param name="kg">The green luma coefficient.</param>
@ -218,11 +242,16 @@ internal static class Av1YuvConverter
/// <param name="lumaBias">The encoded luma bias.</param>
/// <param name="lumaScale">The encoded luma range.</param>
/// <param name="chromaScale">The encoded chroma range.</param>
private static void ConvertYuv444ToRgbRow(
private static void ConvertYuvToRgbRow(
ReadOnlySpan<byte> ySource,
ReadOnlySpan<byte> uSource,
ReadOnlySpan<byte> vSource,
in Buffer2DRegion<byte> uPlane,
in Buffer2DRegion<byte> vPlane,
int yCoordinate,
Span<Rgb24> destination,
bool isMonochrome,
int subX,
int subY,
ObuChromoSamplePosition chromaSamplePosition,
ConversionMode mode,
float kr,
float kg,
@ -234,31 +263,43 @@ internal static class Av1YuvConverter
for (int x = 0; x < destination.Length; x++)
{
float y = (ySource[x] - lumaBias) / lumaScale;
float cb = (uSource[x] - ChromaBias) / chromaScale;
float cr = (vSource[x] - ChromaBias) / chromaScale;
float r;
float g;
float b;
switch (mode)
if (isMonochrome)
{
case ConversionMode.Identity:
// H.273 identity coding stores the nonlinear G, B, and R signals in Y, U, and V order.
r = (vSource[x] - lumaBias) / lumaScale;
g = y;
b = (uSource[x] - lumaBias) / lumaScale;
break;
case ConversionMode.YCgCo:
float temporary = y - cb;
r = temporary + cr;
g = y + cb;
b = temporary - cr;
break;
default:
r = y + (2F * (1F - kr) * cr);
g = y - (2F * ((kr * (1F - kr) * cr) + (kb * (1F - kb) * cb)) / kg);
b = y + (2F * (1F - kb) * cb);
break;
r = y;
g = y;
b = y;
}
else
{
float u = SampleChroma(uPlane, x, yCoordinate, subX, subY, chromaSamplePosition);
float v = SampleChroma(vPlane, x, yCoordinate, subX, subY, chromaSamplePosition);
float cb = (u - ChromaBias) / chromaScale;
float cr = (v - ChromaBias) / chromaScale;
switch (mode)
{
case ConversionMode.Identity:
// H.273 identity coding stores the nonlinear G, B, and R signals in Y, U, and V order.
r = (v - lumaBias) / lumaScale;
g = y;
b = (u - lumaBias) / lumaScale;
break;
case ConversionMode.YCgCo:
float temporary = y - cb;
r = temporary + cr;
g = y + cb;
b = temporary - cr;
break;
default:
r = y + (2F * (1F - kr) * cr);
g = y - (2F * ((kr * (1F - kr) * cr) + (kb * (1F - kb) * cb)) / kg);
b = y + (2F * (1F - kb) * cb);
break;
}
}
destination[x] = new Rgb24(
@ -268,6 +309,80 @@ internal static class Av1YuvConverter
}
}
/// <summary>
/// Bilinearly reconstructs a chroma sample at a luma coordinate.
/// </summary>
/// <param name="plane">The subsampled chroma plane.</param>
/// <param name="x">The luma column coordinate.</param>
/// <param name="y">The luma row coordinate.</param>
/// <param name="subX">The horizontal chroma subsampling shift.</param>
/// <param name="subY">The vertical chroma subsampling shift.</param>
/// <param name="chromaSamplePosition">The spatial position of subsampled chroma.</param>
/// <returns>The reconstructed encoded chroma sample.</returns>
private static float SampleChroma(
in Buffer2DRegion<byte> plane,
int x,
int y,
int subX,
int subY,
ObuChromoSamplePosition chromaSamplePosition)
{
// Unknown 4:2:0 and all 4:2:2 input use the centered convention employed by libavif.
bool isCenteredX = subX != 0 && (subY == 0 || chromaSamplePosition == ObuChromoSamplePosition.Unknown);
bool isCenteredY = subY != 0 && chromaSamplePosition != ObuChromoSamplePosition.Colocated;
GetChromaCoordinates(x, subX, isCenteredX, plane.Width - 1, out int x0, out int x1, out int x1Weight);
GetChromaCoordinates(y, subY, isCenteredY, plane.Height - 1, out int y0, out int y1, out int y1Weight);
ReadOnlySpan<byte> row0 = plane.DangerousGetRowSpan(y0);
ReadOnlySpan<byte> row1 = plane.DangerousGetRowSpan(y1);
int top = (row0[x0] * (4 - x1Weight)) + (row0[x1] * x1Weight);
int bottom = (row1[x0] * (4 - x1Weight)) + (row1[x1] * x1Weight);
return ((top * (4 - y1Weight)) + (bottom * y1Weight)) / 16F;
}
/// <summary>
/// Resolves the two chroma samples and quarter-sample weight surrounding a luma coordinate.
/// </summary>
/// <param name="coordinate">The luma coordinate.</param>
/// <param name="subsampling">The chroma subsampling shift.</param>
/// <param name="isCentered">Whether chroma lies between neighboring luma samples.</param>
/// <param name="maximum">The last available chroma coordinate.</param>
/// <param name="lower">The lower chroma coordinate.</param>
/// <param name="upper">The upper chroma coordinate.</param>
/// <param name="upperWeight">The upper-coordinate weight with a denominator of four.</param>
private static void GetChromaCoordinates(
int coordinate,
int subsampling,
bool isCentered,
int maximum,
out int lower,
out int upper,
out int upperWeight)
{
if (subsampling == 0)
{
lower = coordinate;
upper = coordinate;
upperWeight = 0;
return;
}
int sample = coordinate >> 1;
bool isOdd = (coordinate & 1) != 0;
if (isCentered)
{
lower = isOdd ? sample : Math.Max(sample - 1, 0);
upper = isOdd ? Math.Min(sample + 1, maximum) : sample;
upperWeight = isOdd ? 1 : 3;
return;
}
lower = sample;
upper = isOdd ? Math.Min(sample + 1, maximum) : sample;
upperWeight = isOdd ? 2 : 0;
}
/// <summary>
/// Converts one packed RGB row to YUV 4:4:4 using the resolved H.273 conversion state.
/// </summary>

5
src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs

@ -19,4 +19,9 @@ internal enum ObuChromoSamplePosition : byte
/// Co-located with luma(0, 0) sample
/// </summary>
Colocated = 2,
/// <summary>
/// Reserved and invalid for AV1 content.
/// </summary>
Reserved = 3,
}

113
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs

@ -91,6 +91,110 @@ public class Av1YuvConverterTests
Assert.Equal(b, actual.B, 1d);
}
[Fact]
public void Yuv400ToRgbExpandsLimitedRangeLuma()
{
// Assign
using Image<Rgb24> image = new(2, 1);
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(
2,
1,
false,
ObuMatrixCoefficients.Identity,
Av1ColorFormat.Yuv400);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false);
Span<byte> yRow = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0).DangerousGetRowSpan(0);
yRow[0] = 16;
yRow[1] = 235;
// Act
Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame);
// Assert
Span<Rgb24> actual = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0);
Assert.Equal(new Rgb24(0, 0, 0), actual[0]);
Assert.Equal(new Rgb24(255, 255, 255), actual[1]);
}
[Fact]
public void Yuv422ToRgbBilinearlyUpsamplesCenteredChroma()
{
// Assign
using Image<Rgb24> image = new(4, 1);
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(4, 1, colorFormat: Av1ColorFormat.Yuv422);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv422, false);
frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0).DangerousGetRowSpan(0).Fill(128);
Span<byte> uRow = frameBuffer.DeriveBlockPointer(Av1Plane.U, 1, 0).DangerousGetRowSpan(0);
uRow[0] = 128;
uRow[1] = 192;
frameBuffer.DeriveBlockPointer(Av1Plane.V, 1, 0).DangerousGetRowSpan(0).Fill(128);
// Act
Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame);
// Assert
Span<Rgb24> actual = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0);
Assert.Equal(new Rgb24(128, 128, 128), actual[0]);
Assert.Equal(new Rgb24(128, 125, 158), actual[1]);
Assert.Equal(new Rgb24(128, 119, 217), actual[2]);
Assert.Equal(new Rgb24(128, 116, 247), actual[3]);
}
[Theory]
[InlineData(ObuChromoSamplePosition.Unknown, 158, 98)]
[InlineData(ObuChromoSamplePosition.Vertical, 187, 98)]
[InlineData(ObuChromoSamplePosition.Colocated, 187, 69)]
public void Yuv420ToRgbUsesChromaSamplePosition(int chromaSamplePosition, byte expectedTopBlue, byte expectedLeftBlue)
{
// Assign
using Image<Rgb24> image = new(4, 4);
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(
4,
4,
colorFormat: Av1ColorFormat.Yuv420,
chromaSamplePosition: (ObuChromoSamplePosition)chromaSamplePosition);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false);
Buffer2DRegion<byte> yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0);
Buffer2DRegion<byte> uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 1, 1);
Buffer2DRegion<byte> vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 1, 1);
for (int y = 0; y < yPlane.Height; y++)
{
yPlane.DangerousGetRowSpan(y).Fill(128);
}
uPlane.DangerousGetRowSpan(0)[0] = 128;
uPlane.DangerousGetRowSpan(0)[1] = 192;
uPlane.DangerousGetRowSpan(1)[0] = 64;
uPlane.DangerousGetRowSpan(1)[1] = 255;
vPlane.DangerousGetRowSpan(0).Fill(128);
vPlane.DangerousGetRowSpan(1).Fill(128);
// Act
Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame);
// Assert
Assert.Equal(expectedTopBlue, image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0)[1].B);
Assert.Equal(expectedLeftBlue, image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(1)[0].B);
}
[Fact]
public void Yuv420UsesCeilingChromaPlaneDimensions()
{
// Assign
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(3, 3, colorFormat: Av1ColorFormat.Yuv420);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false);
// Act
Buffer2DRegion<byte> uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 1, 1);
Buffer2DRegion<byte> vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 1, 1);
// Assert
Assert.Equal(new Size(2, 2), uPlane.Size);
Assert.Equal(new Size(2, 2), vPlane.Size);
}
[Fact]
public void RgbToYuvCompareToReferenceRandomPixels()
{
@ -252,17 +356,22 @@ public class Av1YuvConverterTests
int width,
int height,
bool fullRange = true,
ObuMatrixCoefficients matrixCoefficients = ObuMatrixCoefficients.Bt709)
ObuMatrixCoefficients matrixCoefficients = ObuMatrixCoefficients.Bt709,
Av1ColorFormat colorFormat = Av1ColorFormat.Yuv444,
ObuChromoSamplePosition chromaSamplePosition = ObuChromoSamplePosition.Unknown)
=> new()
{
MaxFrameWidth = width,
MaxFrameHeight = height,
ColorConfig = new ObuColorConfig
{
IsMonochrome = false,
IsMonochrome = colorFormat == Av1ColorFormat.Yuv400,
BitDepth = Av1BitDepth.EightBit,
MatrixCoefficients = matrixCoefficients,
ColorRange = fullRange,
SubSamplingX = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422,
SubSamplingY = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420,
ChromaSamplePosition = chromaSamplePosition,
},
};
}

Loading…
Cancel
Save