From 5e2908114dcf6071859c82a1f2a02ec17393cb35 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 24 Aug 2026 17:39:44 +1000 Subject: [PATCH] Support high-bit-depth AV1 color conversion --- HEIF_IMPLEMENTATION_PLAN.md | 5 +- .../Formats/Heif/Av1/Av1FrameBuffer.cs | 171 ++++++---- .../Formats/Heif/Av1/Av1YuvConverter.cs | 297 ++++++++++++------ .../Formats/Heif/Av1/Av1YuvConverterTests.cs | 101 +++++- 4 files changed, 425 insertions(+), 149 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 88e3643dae..27e10dec4b 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -57,6 +57,7 @@ This snapshot pins or classifies the available references and failures; it does | Managed implementation | Normative behavior | Reviewed implementation reference | Use | | --- | --- | --- | --- | | `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. | +| `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. | This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped. @@ -97,8 +98,8 @@ This assessment is based on the current source after the upstream ImageSharp mer - Decoder state is allocated or replaced at multiple points, making parsed tile state, reconstructed frame state, and ownership unclear. - The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements. - Loop restoration, filter intra prediction, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches. -- The active reconstruction path remains limited to an 8-bit byte buffer. Output conversion now handles monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 planes, but the generic frame buffer does not yet establish correct storage and indexing for 10/12-bit samples. -- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8-bit output and uses one allocator-backed RGB row. High-bit-depth conversion, constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete. +- The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active prediction and block reconstruction path is still limited to 8-bit samples and must be connected to the existing high-bit-depth inverse-transform core. +- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output and uses one allocator-backed RGB row. Constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete. - The inverse-transform path allocates arrays in a per-transform hot path. - No usable end-to-end AV1 SIMD path was found. The most visible 4x4 forward-transform SIMD call is commented out, while the production prediction, transform, filter, and output paths are predominantly scalar. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs index d66b2c41e7..a0baba3617 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Memory; @@ -18,6 +20,7 @@ internal class Av1FrameBuffer : IDisposable private const int PictureBufferCrFlag = 1 << 2; private const int PictureBufferLumaMask = PictureBufferYFlag; private const int PictureBufferFullMask = PictureBufferYFlag | PictureBufferCbFlag | PictureBufferCrFlag; + private readonly int storageElementsPerSample; public Av1FrameBuffer(Configuration configuration, ObuSequenceHeader sequenceHeader, Av1ColorFormat maxColorFormat, bool is16BitPipeline) { @@ -26,8 +29,13 @@ internal class Av1FrameBuffer : IDisposable this.MaxHeight = sequenceHeader.MaxFrameHeight; this.BitDepth = sequenceHeader.ColorConfig.BitDepth; this.ColorConfig = sequenceHeader.ColorConfig; - int bitsPerPixel = this.BitDepth > Av1BitDepth.EightBit || is16BitPipeline ? 2 : 1; + this.BytesPerSample = this.BitDepth > Av1BitDepth.EightBit || is16BitPipeline ? 2 : 1; + this.storageElementsPerSample = Math.Max( + (this.BytesPerSample + Unsafe.SizeOf() - 1) / Unsafe.SizeOf(), + 1); + this.ColorFormat = colorFormat; + this.Is16BitPipeline = is16BitPipeline; this.BufferEnableMask = sequenceHeader.ColorConfig.IsMonochrome ? PictureBufferLumaMask : PictureBufferFullMask; int leftPadding = DecoderPaddingValue; @@ -69,17 +77,17 @@ internal class Av1FrameBuffer : IDisposable this.BufferCr = null; if ((this.BufferEnableMask & PictureBufferYFlag) != 0) { - this.BufferY = configuration.MemoryAllocator.Allocate2D(strideY * bitsPerPixel, heightY); + this.BufferY = configuration.MemoryAllocator.Allocate2D(strideY * this.storageElementsPerSample, heightY); } if ((this.BufferEnableMask & PictureBufferCbFlag) != 0) { - this.BufferCb = configuration.MemoryAllocator.Allocate2D(strideChroma * bitsPerPixel, heightChroma); + this.BufferCb = configuration.MemoryAllocator.Allocate2D(strideChroma * this.storageElementsPerSample, heightChroma); } if ((this.BufferEnableMask & PictureBufferCrFlag) != 0) { - this.BufferCr = configuration.MemoryAllocator.Allocate2D(strideChroma * bitsPerPixel, heightChroma); + this.BufferCr = configuration.MemoryAllocator.Allocate2D(strideChroma * this.storageElementsPerSample, heightChroma); } this.BitIncrementY = null; @@ -148,6 +156,11 @@ internal class Av1FrameBuffer : IDisposable /// public Av1BitDepth BitDepth { get; set; } + /// + /// Gets the number of bytes used to store each reconstructed sample. + /// + public int BytesPerSample { get; } + /// /// Gets the color configuration signaled by the AV1 sequence header. /// @@ -205,44 +218,52 @@ internal class Av1FrameBuffer : IDisposable /// public Span DeriveBlockPointer(Av1Plane plane, Point locationInPixels, int subX, int subY, out int stride) { - Span blockReconstructionBuffer; - int blockOffset; - Buffer2D buffer; - - switch (plane) - { - case Av1Plane.Y: - Guard.NotNull(this.BufferY); - buffer = this.BufferY; - stride = buffer.Width; - blockOffset = ((this.OriginY + locationInPixels.Y) * stride) + - (this.OriginX + locationInPixels.X); - break; - case Av1Plane.U: - Guard.NotNull(this.BufferCb); - buffer = this.BufferCb; - stride = buffer.Width; - blockOffset = (((this.OriginY >> subY) + locationInPixels.Y) * stride) + - ((this.OriginX >> subX) + locationInPixels.X); - break; - case Av1Plane.V: - default: - Guard.NotNull(this.BufferCr); - buffer = this.BufferCr; - stride = buffer.Width; - blockOffset = (((this.OriginY >> subY) + locationInPixels.Y) * stride) + - ((this.OriginX >> subX) + locationInPixels.X); - break; - } + this.GetPlaneLayout( + plane, + subX, + subY, + out Buffer2D buffer, + out int originX, + out int originY, + out _, + out _); + + int elementStride = buffer.Width; + stride = elementStride / this.storageElementsPerSample; + int blockOffset = (((originY + locationInPixels.Y) * stride) + originX + locationInPixels.X) * + this.storageElementsPerSample; // Deviation from SVT, return PREVIOUS row in Block Reconstruction Buffer. - blockOffset -= stride; + blockOffset -= elementStride; Guard.MustBeGreaterThanOrEqualTo(blockOffset, 0, nameof(blockOffset)); - blockOffset = (this.BitDepth != Av1BitDepth.EightBit || this.Is16BitPipeline) ? blockOffset << 1 : blockOffset; - blockReconstructionBuffer = buffer.DangerousGetSingleSpan()[blockOffset..]; + return buffer.DangerousGetSingleSpan()[blockOffset..]; + } - return blockReconstructionBuffer; + /// + /// Returns a 16-bit sample span starting one row before the specified block. + /// + /// + /// SVT: svt_aom_derive_blk_pointers + /// + public Span DeriveBlockPointer16(Av1Plane plane, Point locationInPixels, int subX, int subY, out int stride) + { + this.GetPlaneLayout( + plane, + subX, + subY, + out Buffer2D buffer, + out int originX, + out int originY, + out _, + out _); + + stride = buffer.Width / this.storageElementsPerSample; + int blockOffset = ((originY + locationInPixels.Y - 1) * stride) + originX + locationInPixels.X; + Guard.MustBeGreaterThanOrEqualTo(blockOffset, 0, nameof(blockOffset)); + + // High-bit-depth reconstruction uses native 16-bit samples in the byte-backed frame planes. + return MemoryMarshal.Cast(buffer.DangerousGetSingleSpan())[blockOffset..]; } /// @@ -253,39 +274,81 @@ internal class Av1FrameBuffer : IDisposable /// public Buffer2DRegion DeriveBlockPointer(Av1Plane plane, int subX, int subY) { - Rectangle region; - Buffer2D buffer; + this.GetPlaneLayout( + plane, + subX, + subY, + out Buffer2D buffer, + out int originX, + out int originY, + out int width, + out int height); + + Rectangle region = new( + originX * this.storageElementsPerSample, + originY, + width * this.storageElementsPerSample, + height); + + return new Buffer2DRegion(buffer, region); + } + /// + /// Returns one logical row of 16-bit samples from the specified plane. + /// + public Span GetHighBitDepthRowSpan(Av1Plane plane, int row, int subX, int subY) + { + this.GetPlaneLayout( + plane, + subX, + subY, + out Buffer2D buffer, + out int originX, + out int originY, + out int width, + out _); + + Span samples = MemoryMarshal.Cast(buffer.DangerousGetRowSpan(originY + row)); + return samples.Slice(originX, width); + } + + private void GetPlaneLayout( + Av1Plane plane, + int subX, + int subY, + out Buffer2D buffer, + out int originX, + out int originY, + out int width, + out int height) + { switch (plane) { case Av1Plane.Y: Guard.NotNull(this.BufferY); buffer = this.BufferY; - region = new Rectangle(this.OriginX, this.OriginY, this.Width, this.Height); + originX = this.OriginX; + originY = this.OriginY; + width = this.Width; + height = this.Height; break; case Av1Plane.U: Guard.NotNull(this.BufferCb); buffer = this.BufferCb; - region = new Rectangle( - this.OriginX >> subX, - this.OriginY >> subY, - Av1Math.DivideLog2Ceiling(this.Width, subX), - Av1Math.DivideLog2Ceiling(this.Height, subY)); - + originX = this.OriginX >> subX; + originY = this.OriginY >> subY; + width = Av1Math.DivideLog2Ceiling(this.Width, subX); + height = Av1Math.DivideLog2Ceiling(this.Height, subY); break; case Av1Plane.V: default: Guard.NotNull(this.BufferCr); buffer = this.BufferCr; - region = new Rectangle( - this.OriginX >> subX, - this.OriginY >> subY, - Av1Math.DivideLog2Ceiling(this.Width, subX), - Av1Math.DivideLog2Ceiling(this.Height, subY)); - + originX = this.OriginX >> subX; + originY = this.OriginY >> subY; + width = Av1Math.DivideLog2Ceiling(this.Width, subX); + height = Av1Math.DivideLog2Ceiling(this.Height, subY); break; } - - return new Buffer2DRegion(buffer, region); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs index 6a44b50e76..e7ba0d62ea 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs @@ -2,6 +2,8 @@ // Licensed under the Six Labors Split License. using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; @@ -11,12 +13,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1; internal static class Av1YuvConverter { private const float ByteMaximum = byte.MaxValue; - private const float ChromaBias = 128F; - private const float FullRangeLumaScale = byte.MaxValue; - private const float FullRangeChromaScale = byte.MaxValue; - private const float LimitedRangeLumaBias = 16F; - private const float LimitedRangeLumaScale = 219F; - private const float LimitedRangeChromaScale = 224F; private enum ConversionMode { @@ -26,7 +22,7 @@ internal static class Av1YuvConverter } /// - /// Converts the reconstructed 8-bit YUV planes to packed pixels. + /// Converts the reconstructed YUV planes to packed pixels. /// /// The destination pixel type. /// The configuration used for allocation and pixel conversion. @@ -43,7 +39,9 @@ internal static class Av1YuvConverter out float kb, out float lumaBias, out float lumaScale, - out float chromaScale); + out float chromaBias, + out float chromaScale, + out float sampleMaximum); Buffer2DRegion yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0); bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400; @@ -56,23 +54,69 @@ internal static class Av1YuvConverter for (int y = 0; y < image.Height; y++) { - ConvertYuvToRgbRow( - yPlane.DangerousGetRowSpan(y), - uPlane, - vPlane, - y, - rgbRow, - isMonochrome, - subX, - subY, - frameBuffer.ColorConfig.ChromaSamplePosition, - mode, - kr, - kg, - kb, - lumaBias, - lumaScale, - chromaScale); + int y0 = 0; + int y1 = 0; + int y1Weight = 0; + if (!isMonochrome) + { + GetChromaCoordinates( + y, + subY, + subY != 0 && frameBuffer.ColorConfig.ChromaSamplePosition != ObuChromoSamplePosition.Colocated, + uPlane.Height - 1, + out y0, + out y1, + out y1Weight); + } + + if (frameBuffer.BitDepth == Av1BitDepth.EightBit) + { + ConvertYuvToRgbRow( + yPlane.DangerousGetRowSpan(y), + isMonochrome ? default : uPlane.DangerousGetRowSpan(y0), + isMonochrome ? default : uPlane.DangerousGetRowSpan(y1), + isMonochrome ? default : vPlane.DangerousGetRowSpan(y0), + isMonochrome ? default : vPlane.DangerousGetRowSpan(y1), + y1Weight, + rgbRow, + isMonochrome, + subX, + subY, + frameBuffer.ColorConfig.ChromaSamplePosition, + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } + else + { + ConvertYuvToRgbRow( + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0), + isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y0, subX, subY), + isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y1, subX, subY), + isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y0, subX, subY), + isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y1, subX, subY), + y1Weight, + rgbRow, + isMonochrome, + subX, + subY, + frameBuffer.ColorConfig.ChromaSamplePosition, + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } PixelOperations.Instance.FromRgb24( configuration, @@ -82,7 +126,7 @@ internal static class Av1YuvConverter } /// - /// Converts packed pixels to the 8-bit YUV 4:4:4 planes used by the AV1 encoder. + /// Converts packed pixels to the YUV 4:4:4 planes used by the AV1 encoder. /// /// The source pixel type. /// The configuration used for allocation and pixel conversion. @@ -99,7 +143,9 @@ internal static class Av1YuvConverter out float kb, out float lumaBias, out float lumaScale, - out float chromaScale); + out float chromaBias, + out float chromaScale, + out float sampleMaximum); if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444) { @@ -119,18 +165,40 @@ internal static class Av1YuvConverter image.PixelBuffer.DangerousGetRowSpan(y), rgbRow); - ConvertRgbToYuv444Row( - rgbRow, - yPlane.DangerousGetRowSpan(y), - uPlane.DangerousGetRowSpan(y), - vPlane.DangerousGetRowSpan(y), - mode, - kr, - kg, - kb, - lumaBias, - lumaScale, - chromaScale); + if (frameBuffer.BitDepth == Av1BitDepth.EightBit) + { + ConvertRgbToYuv444Row( + rgbRow, + yPlane.DangerousGetRowSpan(y), + uPlane.DangerousGetRowSpan(y), + vPlane.DangerousGetRowSpan(y), + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } + else + { + ConvertRgbToYuv444Row( + rgbRow, + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0), + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y, 0, 0), + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y, 0, 0), + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } } } @@ -144,7 +212,9 @@ internal static class Av1YuvConverter /// The blue luma coefficient. /// The encoded luma bias. /// The encoded luma range. + /// The encoded chroma midpoint. /// The encoded chroma range. + /// The largest encoded sample value. private static void GetConversionParameters( Av1FrameBuffer frameBuffer, out ConversionMode mode, @@ -153,13 +223,10 @@ internal static class Av1YuvConverter out float kb, out float lumaBias, out float lumaScale, - out float chromaScale) + out float chromaBias, + out float chromaScale, + out float sampleMaximum) { - if (frameBuffer.BitDepth != Av1BitDepth.EightBit) - { - throw new NotSupportedException("Only 8-bit AV1 color conversion is currently supported."); - } - mode = ConversionMode.Coefficients; kr = 0F; kb = 0F; @@ -218,18 +285,24 @@ internal static class Av1YuvConverter throw new NotSupportedException("Limited-range AV1 YCgCo color conversion is not currently supported."); } - lumaBias = isFullRange ? 0F : LimitedRangeLumaBias; - lumaScale = isFullRange ? FullRangeLumaScale : LimitedRangeLumaScale; - chromaScale = isFullRange ? FullRangeChromaScale : LimitedRangeChromaScale; + int bitCount = frameBuffer.BitDepth.GetBitCount(); + int depthScale = 1 << (bitCount - 8); + sampleMaximum = (1 << bitCount) - 1; + chromaBias = 128F * depthScale; + lumaBias = isFullRange ? 0F : 16F * depthScale; + lumaScale = isFullRange ? sampleMaximum : 219F * depthScale; + chromaScale = isFullRange ? sampleMaximum : 224F * depthScale; } /// /// Converts one YUV row to packed RGB using the resolved H.273 conversion state. /// /// The luma samples. - /// The blue-difference chroma plane. - /// The red-difference chroma plane. - /// The luma row coordinate. + /// The upper blue-difference chroma row. + /// The lower blue-difference chroma row. + /// The upper red-difference chroma row. + /// The lower red-difference chroma row. + /// The lower chroma-row weight with a denominator of four. /// The destination RGB pixels. /// Whether the frame contains only luma samples. /// The horizontal chroma subsampling shift. @@ -241,12 +314,16 @@ internal static class Av1YuvConverter /// The blue luma coefficient. /// The encoded luma bias. /// The encoded luma range. + /// The encoded chroma midpoint. /// The encoded chroma range. - private static void ConvertYuvToRgbRow( - ReadOnlySpan ySource, - in Buffer2DRegion uPlane, - in Buffer2DRegion vPlane, - int yCoordinate, + /// The largest encoded sample value. + private static void ConvertYuvToRgbRow( + ReadOnlySpan ySource, + ReadOnlySpan uRow0, + ReadOnlySpan uRow1, + ReadOnlySpan vRow0, + ReadOnlySpan vRow1, + int y1Weight, Span destination, bool isMonochrome, int subX, @@ -258,11 +335,14 @@ internal static class Av1YuvConverter float kb, float lumaBias, float lumaScale, - float chromaScale) + float chromaBias, + float chromaScale, + float sampleMaximum) + where TSample : unmanaged { for (int x = 0; x < destination.Length; x++) { - float y = (ySource[x] - lumaBias) / lumaScale; + float y = (GetSample(ySource, x) - lumaBias) / lumaScale; float r; float g; float b; @@ -275,10 +355,10 @@ internal static class Av1YuvConverter } else { - float u = SampleChroma(uPlane, x, yCoordinate, subX, subY, chromaSamplePosition); - float v = SampleChroma(vPlane, x, yCoordinate, subX, subY, chromaSamplePosition); - float cb = (u - ChromaBias) / chromaScale; - float cr = (v - ChromaBias) / chromaScale; + float u = SampleChroma(uRow0, uRow1, x, subX, subY, chromaSamplePosition, y1Weight); + float v = SampleChroma(vRow0, vRow1, x, subX, subY, chromaSamplePosition, y1Weight); + float cb = (u - chromaBias) / chromaScale; + float cr = (v - chromaBias) / chromaScale; switch (mode) { @@ -303,40 +383,39 @@ internal static class Av1YuvConverter } destination[x] = new Rgb24( - ToByte(r * ByteMaximum), - ToByte(g * ByteMaximum), - ToByte(b * ByteMaximum)); + ToSample(r * ByteMaximum, ByteMaximum), + ToSample(g * ByteMaximum, ByteMaximum), + ToSample(b * ByteMaximum, ByteMaximum)); } } /// /// Bilinearly reconstructs a chroma sample at a luma coordinate. /// - /// The subsampled chroma plane. + /// The upper chroma row. + /// The lower chroma row. /// The luma column coordinate. - /// The luma row coordinate. /// The horizontal chroma subsampling shift. /// The vertical chroma subsampling shift. /// The spatial position of subsampled chroma. + /// The lower chroma-row weight with a denominator of four. /// The reconstructed encoded chroma sample. - private static float SampleChroma( - in Buffer2DRegion plane, + private static float SampleChroma( + ReadOnlySpan row0, + ReadOnlySpan row1, int x, - int y, int subX, int subY, - ObuChromoSamplePosition chromaSamplePosition) + ObuChromoSamplePosition chromaSamplePosition, + int y1Weight) + where TSample : unmanaged { // Unknown 4:2:0 and all 4:2:2 input use the centered convention employed by libavif. bool isCenteredX = subX != 0 && (subY == 0 || chromaSamplePosition == ObuChromoSamplePosition.Unknown); - bool isCenteredY = subY != 0 && chromaSamplePosition != ObuChromoSamplePosition.Colocated; - GetChromaCoordinates(x, subX, isCenteredX, plane.Width - 1, out int x0, out int x1, out int x1Weight); - GetChromaCoordinates(y, subY, isCenteredY, plane.Height - 1, out int y0, out int y1, out int y1Weight); + GetChromaCoordinates(x, subX, isCenteredX, row0.Length - 1, out int x0, out int x1, out int x1Weight); - ReadOnlySpan row0 = plane.DangerousGetRowSpan(y0); - ReadOnlySpan row1 = plane.DangerousGetRowSpan(y1); - int top = (row0[x0] * (4 - x1Weight)) + (row0[x1] * x1Weight); - int bottom = (row1[x0] * (4 - x1Weight)) + (row1[x1] * x1Weight); + float top = (GetSample(row0, x0) * (4 - x1Weight)) + (GetSample(row0, x1) * x1Weight); + float bottom = (GetSample(row1, x0) * (4 - x1Weight)) + (GetSample(row1, x1) * x1Weight); return ((top * (4 - y1Weight)) + (bottom * y1Weight)) / 16F; } @@ -396,19 +475,24 @@ internal static class Av1YuvConverter /// The blue luma coefficient. /// The encoded luma bias. /// The encoded luma range. + /// The encoded chroma midpoint. /// The encoded chroma range. - private static void ConvertRgbToYuv444Row( + /// The largest encoded sample value. + private static void ConvertRgbToYuv444Row( ReadOnlySpan source, - Span yDestination, - Span uDestination, - Span vDestination, + Span yDestination, + Span uDestination, + Span vDestination, ConversionMode mode, float kr, float kg, float kb, float lumaBias, float lumaScale, - float chromaScale) + float chromaBias, + float chromaScale, + float sampleMaximum) + where TSample : unmanaged { for (int x = 0; x < source.Length; x++) { @@ -439,25 +523,56 @@ internal static class Av1YuvConverter break; } - yDestination[x] = ToByte((y * lumaScale) + lumaBias); + yDestination[x] = ToSample((y * lumaScale) + lumaBias, sampleMaximum); if (mode == ConversionMode.Identity) { - uDestination[x] = ToByte((cb * lumaScale) + lumaBias); - vDestination[x] = ToByte((cr * lumaScale) + lumaBias); + uDestination[x] = ToSample((cb * lumaScale) + lumaBias, sampleMaximum); + vDestination[x] = ToSample((cr * lumaScale) + lumaBias, sampleMaximum); } else { - uDestination[x] = ToByte((cb * chromaScale) + ChromaBias); - vDestination[x] = ToByte((cr * chromaScale) + ChromaBias); + uDestination[x] = ToSample((cb * chromaScale) + chromaBias, sampleMaximum); + vDestination[x] = ToSample((cr * chromaScale) + chromaBias, sampleMaximum); } } } /// - /// Rounds and clamps a conversion result to an 8-bit sample. + /// Reads an 8-bit or 16-bit unsigned sample without introducing a separate conversion buffer. + /// + /// The encoded sample type. + /// The source samples. + /// The sample index. + /// The sample value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static float GetSample(ReadOnlySpan source, int index) + where TSample : unmanaged + { + ref TSample sample = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), index); + return typeof(TSample) == typeof(byte) + ? Unsafe.As(ref sample) + : Unsafe.As(ref sample); + } + + /// + /// Rounds and clamps a conversion result to the encoded sample range. /// + /// The encoded sample type. /// The conversion result. + /// The largest encoded sample value. /// The bounded sample. - private static byte ToByte(float value) - => (byte)Numerics.Clamp((int)MathF.Round(value, MidpointRounding.AwayFromZero), byte.MinValue, byte.MaxValue); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TSample ToSample(float value, float maximum) + where TSample : unmanaged + { + int sample = Numerics.Clamp((int)MathF.Round(value, MidpointRounding.AwayFromZero), 0, (int)maximum); + if (typeof(TSample) == typeof(byte)) + { + byte result = (byte)sample; + return Unsafe.As(ref result); + } + + ushort highBitDepthResult = (ushort)sample; + return Unsafe.As(ref highBitDepthResult); + } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs index 70badeec5f..db76d4d57f 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs @@ -117,6 +117,64 @@ public class Av1YuvConverterTests Assert.Equal(new Rgb24(255, 255, 255), actual[1]); } + [Theory] + [InlineData(Av1BitDepth.TenBit, true, 0, 1023, 512)] + [InlineData(Av1BitDepth.TenBit, false, 64, 940, 512)] + [InlineData(Av1BitDepth.TwelveBit, true, 0, 4095, 2048)] + [InlineData(Av1BitDepth.TwelveBit, false, 256, 3760, 2048)] + public void HighBitDepthYuvToRgbExpandsSignaledRange( + int bitDepth, + bool fullRange, + ushort black, + ushort white, + ushort neutralChroma) + { + // Assign + using Image image = new(2, 1); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(2, 1, fullRange, bitDepth: (Av1BitDepth)bitDepth); + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + Span yRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, 0, 0, 0); + yRow[0] = black; + yRow[1] = white; + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, 0, 0, 0).Fill(neutralChroma); + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, 0, 0, 0).Fill(neutralChroma); + + // Act + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame); + + // Assert + Span actual = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0); + Assert.Equal(new Rgb24(0, 0, 0), actual[0]); + Assert.Equal(new Rgb24(255, 255, 255), actual[1]); + } + + [Theory] + [InlineData(Av1BitDepth.TenBit)] + [InlineData(Av1BitDepth.TwelveBit)] + public void HighBitDepthFrameBufferUsesSampleUnitStrides(int bitDepth) + { + // Assign + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + 3, + 3, + colorFormat: Av1ColorFormat.Yuv420, + bitDepth: (Av1BitDepth)bitDepth); + + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false); + + // Act + Span block = frameBuffer.DeriveBlockPointer16(Av1Plane.Y, Point.Empty, 0, 0, out int stride); + block[stride] = 321; + Span chromaRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, 0, 1, 1); + + // Assert + Assert.Equal(2, frameBuffer.BytesPerSample); + Assert.Equal(3 + 144, stride); + Assert.Equal(stride * 2, frameBuffer.BufferY!.Width); + Assert.Equal(321, frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, 0, 0, 0)[0]); + Assert.Equal(2, chromaRow.Length); + } + [Fact] public void Yuv422ToRgbBilinearlyUpsamplesCenteredChroma() { @@ -333,6 +391,44 @@ public class Av1YuvConverterTests Assert.Equal(b, actualPixel.B, 2d); } + [Theory] + [InlineData(Av1BitDepth.TenBit, ObuMatrixCoefficients.Bt709)] + [InlineData(Av1BitDepth.TenBit, ObuMatrixCoefficients.Identity)] + [InlineData(Av1BitDepth.TenBit, ObuMatrixCoefficients.SmpteYCgCo)] + [InlineData(Av1BitDepth.TwelveBit, ObuMatrixCoefficients.Bt709)] + [InlineData(Av1BitDepth.TwelveBit, ObuMatrixCoefficients.Identity)] + [InlineData(Av1BitDepth.TwelveBit, ObuMatrixCoefficients.SmpteYCgCo)] + public void HighBitDepthRoundTrip(int bitDepth, int matrixCoefficients) + { + // Assign + using Image image = new(3, 1); + Span source = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0); + source[0] = new Rgb24(0, 0, 0); + source[1] = new Rgb24(150, 100, 50); + source[2] = new Rgb24(255, 255, 255); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + image.Width, + image.Height, + matrixCoefficients: (ObuMatrixCoefficients)matrixCoefficients, + bitDepth: (Av1BitDepth)bitDepth); + + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + using Image actual = new(image.Width, image.Height); + + // Act + Av1YuvConverter.ConvertFromRgb(Configuration.Default, image.Frames.RootFrame, frameBuffer); + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, actual.Frames.RootFrame); + + // Assert + Span actualPixels = actual.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0); + for (int x = 0; x < source.Length; x++) + { + Assert.Equal(source[x].R, actualPixels[x].R, 1D); + Assert.Equal(source[x].G, actualPixels[x].G, 1D); + Assert.Equal(source[x].B, actualPixels[x].B, 1D); + } + } + // [Theory] // [WithFile(TestImages.Jpeg.Baseline.Winter444_Interleaved, PixelTypes.Rgb24)] public void RoundTrip(TestImageProvider provider) @@ -358,7 +454,8 @@ public class Av1YuvConverterTests bool fullRange = true, ObuMatrixCoefficients matrixCoefficients = ObuMatrixCoefficients.Bt709, Av1ColorFormat colorFormat = Av1ColorFormat.Yuv444, - ObuChromoSamplePosition chromaSamplePosition = ObuChromoSamplePosition.Unknown) + ObuChromoSamplePosition chromaSamplePosition = ObuChromoSamplePosition.Unknown, + Av1BitDepth bitDepth = Av1BitDepth.EightBit) => new() { MaxFrameWidth = width, @@ -366,7 +463,7 @@ public class Av1YuvConverterTests ColorConfig = new ObuColorConfig { IsMonochrome = colorFormat == Av1ColorFormat.Yuv400, - BitDepth = Av1BitDepth.EightBit, + BitDepth = bitDepth, MatrixCoefficients = matrixCoefficients, ColorRange = fullRange, SubSamplingX = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422,