tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Support high-bit-depth AV1 color conversion

pull/2633/head
James Jackson-South 2 months ago
parent
61b9cd4cb6
commit
5e2908114d
4 changed files with 425 additions and 149 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      HEIF_IMPLEMENTATION_PLAN.md
  2. 171
      src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs
  3. 297
      src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs
  4. 101
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs

5
HEIF_IMPLEMENTATION_PLAN.md
View File

@ -57,6 +57,7 @@ This snapshot pins or classifies the available references and failures; it does
| Managed implementation | Normative behavior | Reviewed implementation reference | Use |
| --- | --- | --- | --- |
| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. |
| `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. |
This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped.
@ -97,8 +98,8 @@ This assessment is based on the current source after the upstream ImageSharp mer
- Decoder state is allocated or replaced at multiple points, making parsed tile state, reconstructed frame state, and ownership unclear.
- The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements.
- Loop restoration, filter intra prediction, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches.
- The active reconstruction path remains limited to an 8-bit byte buffer. Output conversion now handles monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 planes, but the generic frame buffer does not yet establish correct storage and indexing for 10/12-bit samples.
- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8-bit output and uses one allocator-backed RGB row. High-bit-depth conversion, constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete.
- The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active prediction and block reconstruction path is still limited to 8-bit samples and must be connected to the existing high-bit-depth inverse-transform core.
- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output and uses one allocator-backed RGB row. Constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete.
- The inverse-transform path allocates arrays in a per-transform hot path.
- No usable end-to-end AV1 SIMD path was found. The most visible 4x4 forward-transform SIMD call is commented out, while the production prediction, transform, filter, and output paths are predominantly scalar.

171
src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs
View File

@ -1,6 +1,8 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Memory;
@ -18,6 +20,7 @@ internal class Av1FrameBuffer<T> : IDisposable
private const int PictureBufferCrFlag = 1 << 2;
private const int PictureBufferLumaMask = PictureBufferYFlag;
private const int PictureBufferFullMask = PictureBufferYFlag | PictureBufferCbFlag | PictureBufferCrFlag;
private readonly int storageElementsPerSample;
public Av1FrameBuffer(Configuration configuration, ObuSequenceHeader sequenceHeader, Av1ColorFormat maxColorFormat, bool is16BitPipeline)
{
@ -26,8 +29,13 @@ internal class Av1FrameBuffer<T> : IDisposable
this.MaxHeight = sequenceHeader.MaxFrameHeight;
this.BitDepth = sequenceHeader.ColorConfig.BitDepth;
this.ColorConfig = sequenceHeader.ColorConfig;
int bitsPerPixel = this.BitDepth > Av1BitDepth.EightBit || is16BitPipeline ? 2 : 1;
this.BytesPerSample = this.BitDepth > Av1BitDepth.EightBit || is16BitPipeline ? 2 : 1;
this.storageElementsPerSample = Math.Max(
(this.BytesPerSample + Unsafe.SizeOf<T>() - 1) / Unsafe.SizeOf<T>(),
1);
this.ColorFormat = colorFormat;
this.Is16BitPipeline = is16BitPipeline;
this.BufferEnableMask = sequenceHeader.ColorConfig.IsMonochrome ? PictureBufferLumaMask : PictureBufferFullMask;
int leftPadding = DecoderPaddingValue;
@ -69,17 +77,17 @@ internal class Av1FrameBuffer<T> : IDisposable
this.BufferCr = null;
if ((this.BufferEnableMask & PictureBufferYFlag) != 0)
{
this.BufferY = configuration.MemoryAllocator.Allocate2D<T>(strideY * bitsPerPixel, heightY);
this.BufferY = configuration.MemoryAllocator.Allocate2D<T>(strideY * this.storageElementsPerSample, heightY);
}
if ((this.BufferEnableMask & PictureBufferCbFlag) != 0)
{
this.BufferCb = configuration.MemoryAllocator.Allocate2D<T>(strideChroma * bitsPerPixel, heightChroma);
this.BufferCb = configuration.MemoryAllocator.Allocate2D<T>(strideChroma * this.storageElementsPerSample, heightChroma);
}
if ((this.BufferEnableMask & PictureBufferCrFlag) != 0)
{
this.BufferCr = configuration.MemoryAllocator.Allocate2D<T>(strideChroma * bitsPerPixel, heightChroma);
this.BufferCr = configuration.MemoryAllocator.Allocate2D<T>(strideChroma * this.storageElementsPerSample, heightChroma);
}
this.BitIncrementY = null;
@ -148,6 +156,11 @@ internal class Av1FrameBuffer<T> : IDisposable
/// </summary>
public Av1BitDepth BitDepth { get; set; }
/// <summary>
/// Gets the number of bytes used to store each reconstructed sample.
/// </summary>
public int BytesPerSample { get; }
/// <summary>
/// Gets the color configuration signaled by the AV1 sequence header.
/// </summary>
@ -205,44 +218,52 @@ internal class Av1FrameBuffer<T> : IDisposable
/// </remarks>
public Span<T> DeriveBlockPointer(Av1Plane plane, Point locationInPixels, int subX, int subY, out int stride)
{
Span<T> blockReconstructionBuffer;
int blockOffset;
Buffer2D<T> buffer;
switch (plane)
{
case Av1Plane.Y:
Guard.NotNull(this.BufferY);
buffer = this.BufferY;
stride = buffer.Width;
blockOffset = ((this.OriginY + locationInPixels.Y) * stride) +
(this.OriginX + locationInPixels.X);
break;
case Av1Plane.U:
Guard.NotNull(this.BufferCb);
buffer = this.BufferCb;
stride = buffer.Width;
blockOffset = (((this.OriginY >> subY) + locationInPixels.Y) * stride) +
((this.OriginX >> subX) + locationInPixels.X);
break;
case Av1Plane.V:
default:
Guard.NotNull(this.BufferCr);
buffer = this.BufferCr;
stride = buffer.Width;
blockOffset = (((this.OriginY >> subY) + locationInPixels.Y) * stride) +
((this.OriginX >> subX) + locationInPixels.X);
break;
}
this.GetPlaneLayout(
plane,
subX,
subY,
out Buffer2D<T> buffer,
out int originX,
out int originY,
out _,
out _);
int elementStride = buffer.Width;
stride = elementStride / this.storageElementsPerSample;
int blockOffset = (((originY + locationInPixels.Y) * stride) + originX + locationInPixels.X) *
this.storageElementsPerSample;
// Deviation from SVT, return PREVIOUS row in Block Reconstruction Buffer.
blockOffset -= stride;
blockOffset -= elementStride;
Guard.MustBeGreaterThanOrEqualTo(blockOffset, 0, nameof(blockOffset));
blockOffset = (this.BitDepth != Av1BitDepth.EightBit || this.Is16BitPipeline) ? blockOffset << 1 : blockOffset;
blockReconstructionBuffer = buffer.DangerousGetSingleSpan()[blockOffset..];
return buffer.DangerousGetSingleSpan()[blockOffset..];
}
return blockReconstructionBuffer;
/// <summary>
/// Returns a 16-bit sample span starting one row before the specified block.
/// </summary>
/// <remarks>
/// SVT: svt_aom_derive_blk_pointers
/// </remarks>
public Span<short> DeriveBlockPointer16(Av1Plane plane, Point locationInPixels, int subX, int subY, out int stride)
{
this.GetPlaneLayout(
plane,
subX,
subY,
out Buffer2D<T> buffer,
out int originX,
out int originY,
out _,
out _);
stride = buffer.Width / this.storageElementsPerSample;
int blockOffset = ((originY + locationInPixels.Y - 1) * stride) + originX + locationInPixels.X;
Guard.MustBeGreaterThanOrEqualTo(blockOffset, 0, nameof(blockOffset));
// High-bit-depth reconstruction uses native 16-bit samples in the byte-backed frame planes.
return MemoryMarshal.Cast<T, short>(buffer.DangerousGetSingleSpan())[blockOffset..];
}
/// <summary>
@ -253,39 +274,81 @@ internal class Av1FrameBuffer<T> : IDisposable
/// </remarks>
public Buffer2DRegion<T> DeriveBlockPointer(Av1Plane plane, int subX, int subY)
{
Rectangle region;
Buffer2D<T> buffer;
this.GetPlaneLayout(
plane,
subX,
subY,
out Buffer2D<T> buffer,
out int originX,
out int originY,
out int width,
out int height);
Rectangle region = new(
originX * this.storageElementsPerSample,
originY,
width * this.storageElementsPerSample,
height);
return new Buffer2DRegion<T>(buffer, region);
}
/// <summary>
/// Returns one logical row of 16-bit samples from the specified plane.
/// </summary>
public Span<ushort> GetHighBitDepthRowSpan(Av1Plane plane, int row, int subX, int subY)
{
this.GetPlaneLayout(
plane,
subX,
subY,
out Buffer2D<T> buffer,
out int originX,
out int originY,
out int width,
out _);
Span<ushort> samples = MemoryMarshal.Cast<T, ushort>(buffer.DangerousGetRowSpan(originY + row));
return samples.Slice(originX, width);
}
private void GetPlaneLayout(
Av1Plane plane,
int subX,
int subY,
out Buffer2D<T> buffer,
out int originX,
out int originY,
out int width,
out int height)
{
switch (plane)
{
case Av1Plane.Y:
Guard.NotNull(this.BufferY);
buffer = this.BufferY;
region = new Rectangle(this.OriginX, this.OriginY, this.Width, this.Height);
originX = this.OriginX;
originY = this.OriginY;
width = this.Width;
height = this.Height;
break;
case Av1Plane.U:
Guard.NotNull(this.BufferCb);
buffer = this.BufferCb;
region = new Rectangle(
this.OriginX >> subX,
this.OriginY >> subY,
Av1Math.DivideLog2Ceiling(this.Width, subX),
Av1Math.DivideLog2Ceiling(this.Height, subY));
originX = this.OriginX >> subX;
originY = this.OriginY >> subY;
width = Av1Math.DivideLog2Ceiling(this.Width, subX);
height = Av1Math.DivideLog2Ceiling(this.Height, subY);
break;
case Av1Plane.V:
default:
Guard.NotNull(this.BufferCr);
buffer = this.BufferCr;
region = new Rectangle(
this.OriginX >> subX,
this.OriginY >> subY,
Av1Math.DivideLog2Ceiling(this.Width, subX),
Av1Math.DivideLog2Ceiling(this.Height, subY));
originX = this.OriginX >> subX;
originY = this.OriginY >> subY;
width = Av1Math.DivideLog2Ceiling(this.Width, subX);
height = Av1Math.DivideLog2Ceiling(this.Height, subY);
break;
}
return new Buffer2DRegion<T>(buffer, region);
}
}

297
src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs
View File

@ -2,6 +2,8 @@
// Licensed under the Six Labors Split License.
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats;
@ -11,12 +13,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1;
internal static class Av1YuvConverter
{
private const float ByteMaximum = byte.MaxValue;
private const float ChromaBias = 128F;
private const float FullRangeLumaScale = byte.MaxValue;
private const float FullRangeChromaScale = byte.MaxValue;
private const float LimitedRangeLumaBias = 16F;
private const float LimitedRangeLumaScale = 219F;
private const float LimitedRangeChromaScale = 224F;
private enum ConversionMode
{
@ -26,7 +22,7 @@ internal static class Av1YuvConverter
}
/// <summary>
/// Converts the reconstructed 8-bit YUV planes to packed pixels.
/// Converts the reconstructed YUV planes to packed pixels.
/// </summary>
/// <typeparam name="TPixel">The destination pixel type.</typeparam>
/// <param name="configuration">The configuration used for allocation and pixel conversion.</param>
@ -43,7 +39,9 @@ internal static class Av1YuvConverter
out float kb,
out float lumaBias,
out float lumaScale,
out float chromaScale);
out float chromaBias,
out float chromaScale,
out float sampleMaximum);
Buffer2DRegion<byte> yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0);
bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400;
@ -56,23 +54,69 @@ internal static class Av1YuvConverter
for (int y = 0; y < image.Height; y++)
{
ConvertYuvToRgbRow(
yPlane.DangerousGetRowSpan(y),
uPlane,
vPlane,
y,
rgbRow,
isMonochrome,
subX,
subY,
frameBuffer.ColorConfig.ChromaSamplePosition,
mode,
kr,
kg,
kb,
lumaBias,
lumaScale,
chromaScale);
int y0 = 0;
int y1 = 0;
int y1Weight = 0;
if (!isMonochrome)
{
GetChromaCoordinates(
y,
subY,
subY != 0 && frameBuffer.ColorConfig.ChromaSamplePosition != ObuChromoSamplePosition.Colocated,
uPlane.Height - 1,
out y0,
out y1,
out y1Weight);
}
if (frameBuffer.BitDepth == Av1BitDepth.EightBit)
{
ConvertYuvToRgbRow(
yPlane.DangerousGetRowSpan(y),
isMonochrome ? default : uPlane.DangerousGetRowSpan(y0),
isMonochrome ? default : uPlane.DangerousGetRowSpan(y1),
isMonochrome ? default : vPlane.DangerousGetRowSpan(y0),
isMonochrome ? default : vPlane.DangerousGetRowSpan(y1),
y1Weight,
rgbRow,
isMonochrome,
subX,
subY,
frameBuffer.ColorConfig.ChromaSamplePosition,
mode,
kr,
kg,
kb,
lumaBias,
lumaScale,
chromaBias,
chromaScale,
sampleMaximum);
}
else
{
ConvertYuvToRgbRow(
frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0),
isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y0, subX, subY),
isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y1, subX, subY),
isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y0, subX, subY),
isMonochrome ? default : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y1, subX, subY),
y1Weight,
rgbRow,
isMonochrome,
subX,
subY,
frameBuffer.ColorConfig.ChromaSamplePosition,
mode,
kr,
kg,
kb,
lumaBias,
lumaScale,
chromaBias,
chromaScale,
sampleMaximum);
}
PixelOperations<TPixel>.Instance.FromRgb24(
configuration,
@ -82,7 +126,7 @@ internal static class Av1YuvConverter
}
/// <summary>
/// Converts packed pixels to the 8-bit YUV 4:4:4 planes used by the AV1 encoder.
/// Converts packed pixels to the YUV 4:4:4 planes used by the AV1 encoder.
/// </summary>
/// <typeparam name="TPixel">The source pixel type.</typeparam>
/// <param name="configuration">The configuration used for allocation and pixel conversion.</param>
@ -99,7 +143,9 @@ internal static class Av1YuvConverter
out float kb,
out float lumaBias,
out float lumaScale,
out float chromaScale);
out float chromaBias,
out float chromaScale,
out float sampleMaximum);
if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444)
{
@ -119,18 +165,40 @@ internal static class Av1YuvConverter
image.PixelBuffer.DangerousGetRowSpan(y),
rgbRow);
ConvertRgbToYuv444Row(
rgbRow,
yPlane.DangerousGetRowSpan(y),
uPlane.DangerousGetRowSpan(y),
vPlane.DangerousGetRowSpan(y),
mode,
kr,
kg,
kb,
lumaBias,
lumaScale,
chromaScale);
if (frameBuffer.BitDepth == Av1BitDepth.EightBit)
{
ConvertRgbToYuv444Row(
rgbRow,
yPlane.DangerousGetRowSpan(y),
uPlane.DangerousGetRowSpan(y),
vPlane.DangerousGetRowSpan(y),
mode,
kr,
kg,
kb,
lumaBias,
lumaScale,
chromaBias,
chromaScale,
sampleMaximum);
}
else
{
ConvertRgbToYuv444Row(
rgbRow,
frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0),
frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y, 0, 0),
frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y, 0, 0),
mode,
kr,
kg,
kb,
lumaBias,
lumaScale,
chromaBias,
chromaScale,
sampleMaximum);
}
}
}
@ -144,7 +212,9 @@ internal static class Av1YuvConverter
/// <param name="kb">The blue luma coefficient.</param>
/// <param name="lumaBias">The encoded luma bias.</param>
/// <param name="lumaScale">The encoded luma range.</param>
/// <param name="chromaBias">The encoded chroma midpoint.</param>
/// <param name="chromaScale">The encoded chroma range.</param>
/// <param name="sampleMaximum">The largest encoded sample value.</param>
private static void GetConversionParameters(
Av1FrameBuffer<byte> frameBuffer,
out ConversionMode mode,
@ -153,13 +223,10 @@ internal static class Av1YuvConverter
out float kb,
out float lumaBias,
out float lumaScale,
out float chromaScale)
out float chromaBias,
out float chromaScale,
out float sampleMaximum)
{
if (frameBuffer.BitDepth != Av1BitDepth.EightBit)
{
throw new NotSupportedException("Only 8-bit AV1 color conversion is currently supported.");
}
mode = ConversionMode.Coefficients;
kr = 0F;
kb = 0F;
@ -218,18 +285,24 @@ internal static class Av1YuvConverter
throw new NotSupportedException("Limited-range AV1 YCgCo color conversion is not currently supported.");
}
lumaBias = isFullRange ? 0F : LimitedRangeLumaBias;
lumaScale = isFullRange ? FullRangeLumaScale : LimitedRangeLumaScale;
chromaScale = isFullRange ? FullRangeChromaScale : LimitedRangeChromaScale;
int bitCount = frameBuffer.BitDepth.GetBitCount();
int depthScale = 1 << (bitCount - 8);
sampleMaximum = (1 << bitCount) - 1;
chromaBias = 128F * depthScale;
lumaBias = isFullRange ? 0F : 16F * depthScale;
lumaScale = isFullRange ? sampleMaximum : 219F * depthScale;
chromaScale = isFullRange ? sampleMaximum : 224F * depthScale;
}
/// <summary>
/// Converts one YUV row to packed RGB using the resolved H.273 conversion state.
/// </summary>
/// <param name="ySource">The luma samples.</param>
/// <param name="uPlane">The blue-difference chroma plane.</param>
/// <param name="vPlane">The red-difference chroma plane.</param>
/// <param name="yCoordinate">The luma row coordinate.</param>
/// <param name="uRow0">The upper blue-difference chroma row.</param>
/// <param name="uRow1">The lower blue-difference chroma row.</param>
/// <param name="vRow0">The upper red-difference chroma row.</param>
/// <param name="vRow1">The lower red-difference chroma row.</param>
/// <param name="y1Weight">The lower chroma-row weight with a denominator of four.</param>
/// <param name="destination">The destination RGB pixels.</param>
/// <param name="isMonochrome">Whether the frame contains only luma samples.</param>
/// <param name="subX">The horizontal chroma subsampling shift.</param>
@ -241,12 +314,16 @@ internal static class Av1YuvConverter
/// <param name="kb">The blue luma coefficient.</param>
/// <param name="lumaBias">The encoded luma bias.</param>
/// <param name="lumaScale">The encoded luma range.</param>
/// <param name="chromaBias">The encoded chroma midpoint.</param>
/// <param name="chromaScale">The encoded chroma range.</param>
private static void ConvertYuvToRgbRow(
ReadOnlySpan<byte> ySource,
in Buffer2DRegion<byte> uPlane,
in Buffer2DRegion<byte> vPlane,
int yCoordinate,
/// <param name="sampleMaximum">The largest encoded sample value.</param>
private static void ConvertYuvToRgbRow<TSample>(
ReadOnlySpan<TSample> ySource,
ReadOnlySpan<TSample> uRow0,
ReadOnlySpan<TSample> uRow1,
ReadOnlySpan<TSample> vRow0,
ReadOnlySpan<TSample> vRow1,
int y1Weight,
Span<Rgb24> destination,
bool isMonochrome,
int subX,
@ -258,11 +335,14 @@ internal static class Av1YuvConverter
float kb,
float lumaBias,
float lumaScale,
float chromaScale)
float chromaBias,
float chromaScale,
float sampleMaximum)
where TSample : unmanaged
{
for (int x = 0; x < destination.Length; x++)
{
float y = (ySource[x] - lumaBias) / lumaScale;
float y = (GetSample(ySource, x) - lumaBias) / lumaScale;
float r;
float g;
float b;
@ -275,10 +355,10 @@ internal static class Av1YuvConverter
}
else
{
float u = SampleChroma(uPlane, x, yCoordinate, subX, subY, chromaSamplePosition);
float v = SampleChroma(vPlane, x, yCoordinate, subX, subY, chromaSamplePosition);
float cb = (u - ChromaBias) / chromaScale;
float cr = (v - ChromaBias) / chromaScale;
float u = SampleChroma(uRow0, uRow1, x, subX, subY, chromaSamplePosition, y1Weight);
float v = SampleChroma(vRow0, vRow1, x, subX, subY, chromaSamplePosition, y1Weight);
float cb = (u - chromaBias) / chromaScale;
float cr = (v - chromaBias) / chromaScale;
switch (mode)
{
@ -303,40 +383,39 @@ internal static class Av1YuvConverter
}
destination[x] = new Rgb24(
ToByte(r * ByteMaximum),
ToByte(g * ByteMaximum),
ToByte(b * ByteMaximum));
ToSample<byte>(r * ByteMaximum, ByteMaximum),
ToSample<byte>(g * ByteMaximum, ByteMaximum),
ToSample<byte>(b * ByteMaximum, ByteMaximum));
}
}
/// <summary>
/// Bilinearly reconstructs a chroma sample at a luma coordinate.
/// </summary>
/// <param name="plane">The subsampled chroma plane.</param>
/// <param name="row0">The upper chroma row.</param>
/// <param name="row1">The lower chroma row.</param>
/// <param name="x">The luma column coordinate.</param>
/// <param name="y">The luma row coordinate.</param>
/// <param name="subX">The horizontal chroma subsampling shift.</param>
/// <param name="subY">The vertical chroma subsampling shift.</param>
/// <param name="chromaSamplePosition">The spatial position of subsampled chroma.</param>
/// <param name="y1Weight">The lower chroma-row weight with a denominator of four.</param>
/// <returns>The reconstructed encoded chroma sample.</returns>
private static float SampleChroma(
in Buffer2DRegion<byte> plane,
private static float SampleChroma<TSample>(
ReadOnlySpan<TSample> row0,
ReadOnlySpan<TSample> row1,
int x,
int y,
int subX,
int subY,
ObuChromoSamplePosition chromaSamplePosition)
ObuChromoSamplePosition chromaSamplePosition,
int y1Weight)
where TSample : unmanaged
{
// Unknown 4:2:0 and all 4:2:2 input use the centered convention employed by libavif.
bool isCenteredX = subX != 0 && (subY == 0 || chromaSamplePosition == ObuChromoSamplePosition.Unknown);
bool isCenteredY = subY != 0 && chromaSamplePosition != ObuChromoSamplePosition.Colocated;
GetChromaCoordinates(x, subX, isCenteredX, plane.Width - 1, out int x0, out int x1, out int x1Weight);
GetChromaCoordinates(y, subY, isCenteredY, plane.Height - 1, out int y0, out int y1, out int y1Weight);
GetChromaCoordinates(x, subX, isCenteredX, row0.Length - 1, out int x0, out int x1, out int x1Weight);
ReadOnlySpan<byte> row0 = plane.DangerousGetRowSpan(y0);
ReadOnlySpan<byte> row1 = plane.DangerousGetRowSpan(y1);
int top = (row0[x0] * (4 - x1Weight)) + (row0[x1] * x1Weight);
int bottom = (row1[x0] * (4 - x1Weight)) + (row1[x1] * x1Weight);
float top = (GetSample(row0, x0) * (4 - x1Weight)) + (GetSample(row0, x1) * x1Weight);
float bottom = (GetSample(row1, x0) * (4 - x1Weight)) + (GetSample(row1, x1) * x1Weight);
return ((top * (4 - y1Weight)) + (bottom * y1Weight)) / 16F;
}
@ -396,19 +475,24 @@ internal static class Av1YuvConverter
/// <param name="kb">The blue luma coefficient.</param>
/// <param name="lumaBias">The encoded luma bias.</param>
/// <param name="lumaScale">The encoded luma range.</param>
/// <param name="chromaBias">The encoded chroma midpoint.</param>
/// <param name="chromaScale">The encoded chroma range.</param>
private static void ConvertRgbToYuv444Row(
/// <param name="sampleMaximum">The largest encoded sample value.</param>
private static void ConvertRgbToYuv444Row<TSample>(
ReadOnlySpan<Rgb24> source,
Span<byte> yDestination,
Span<byte> uDestination,
Span<byte> vDestination,
Span<TSample> yDestination,
Span<TSample> uDestination,
Span<TSample> vDestination,
ConversionMode mode,
float kr,
float kg,
float kb,
float lumaBias,
float lumaScale,
float chromaScale)
float chromaBias,
float chromaScale,
float sampleMaximum)
where TSample : unmanaged
{
for (int x = 0; x < source.Length; x++)
{
@ -439,25 +523,56 @@ internal static class Av1YuvConverter
break;
}
yDestination[x] = ToByte((y * lumaScale) + lumaBias);
yDestination[x] = ToSample<TSample>((y * lumaScale) + lumaBias, sampleMaximum);
if (mode == ConversionMode.Identity)
{
uDestination[x] = ToByte((cb * lumaScale) + lumaBias);
vDestination[x] = ToByte((cr * lumaScale) + lumaBias);
uDestination[x] = ToSample<TSample>((cb * lumaScale) + lumaBias, sampleMaximum);
vDestination[x] = ToSample<TSample>((cr * lumaScale) + lumaBias, sampleMaximum);
}
else
{
uDestination[x] = ToByte((cb * chromaScale) + ChromaBias);
vDestination[x] = ToByte((cr * chromaScale) + ChromaBias);
uDestination[x] = ToSample<TSample>((cb * chromaScale) + chromaBias, sampleMaximum);
vDestination[x] = ToSample<TSample>((cr * chromaScale) + chromaBias, sampleMaximum);
}
}
}
/// <summary>
/// Rounds and clamps a conversion result to an 8-bit sample.
/// Reads an 8-bit or 16-bit unsigned sample without introducing a separate conversion buffer.
/// </summary>
/// <typeparam name="TSample">The encoded sample type.</typeparam>
/// <param name="source">The source samples.</param>
/// <param name="index">The sample index.</param>
/// <returns>The sample value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static float GetSample<TSample>(ReadOnlySpan<TSample> source, int index)
where TSample : unmanaged
{
ref TSample sample = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), index);
return typeof(TSample) == typeof(byte)
? Unsafe.As<TSample, byte>(ref sample)
: Unsafe.As<TSample, ushort>(ref sample);
}
/// <summary>
/// Rounds and clamps a conversion result to the encoded sample range.
/// </summary>
/// <typeparam name="TSample">The encoded sample type.</typeparam>
/// <param name="value">The conversion result.</param>
/// <param name="maximum">The largest encoded sample value.</param>
/// <returns>The bounded sample.</returns>
private static byte ToByte(float value)
=> (byte)Numerics.Clamp((int)MathF.Round(value, MidpointRounding.AwayFromZero), byte.MinValue, byte.MaxValue);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static TSample ToSample<TSample>(float value, float maximum)
where TSample : unmanaged
{
int sample = Numerics.Clamp((int)MathF.Round(value, MidpointRounding.AwayFromZero), 0, (int)maximum);
if (typeof(TSample) == typeof(byte))
{
byte result = (byte)sample;
return Unsafe.As<byte, TSample>(ref result);
}
ushort highBitDepthResult = (ushort)sample;
return Unsafe.As<ushort, TSample>(ref highBitDepthResult);
}
}

101
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs
View File

@ -117,6 +117,64 @@ public class Av1YuvConverterTests
Assert.Equal(new Rgb24(255, 255, 255), actual[1]);
}
[Theory]
[InlineData(Av1BitDepth.TenBit, true, 0, 1023, 512)]
[InlineData(Av1BitDepth.TenBit, false, 64, 940, 512)]
[InlineData(Av1BitDepth.TwelveBit, true, 0, 4095, 2048)]
[InlineData(Av1BitDepth.TwelveBit, false, 256, 3760, 2048)]
public void HighBitDepthYuvToRgbExpandsSignaledRange(
int bitDepth,
bool fullRange,
ushort black,
ushort white,
ushort neutralChroma)
{
// Assign
using Image<Rgb24> image = new(2, 1);
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(2, 1, fullRange, bitDepth: (Av1BitDepth)bitDepth);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false);
Span<ushort> yRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, 0, 0, 0);
yRow[0] = black;
yRow[1] = white;
frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, 0, 0, 0).Fill(neutralChroma);
frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, 0, 0, 0).Fill(neutralChroma);
// Act
Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, image.Frames.RootFrame);
// Assert
Span<Rgb24> actual = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0);
Assert.Equal(new Rgb24(0, 0, 0), actual[0]);
Assert.Equal(new Rgb24(255, 255, 255), actual[1]);
}
[Theory]
[InlineData(Av1BitDepth.TenBit)]
[InlineData(Av1BitDepth.TwelveBit)]
public void HighBitDepthFrameBufferUsesSampleUnitStrides(int bitDepth)
{
// Assign
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(
3,
3,
colorFormat: Av1ColorFormat.Yuv420,
bitDepth: (Av1BitDepth)bitDepth);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false);
// Act
Span<short> block = frameBuffer.DeriveBlockPointer16(Av1Plane.Y, Point.Empty, 0, 0, out int stride);
block[stride] = 321;
Span<ushort> chromaRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, 0, 1, 1);
// Assert
Assert.Equal(2, frameBuffer.BytesPerSample);
Assert.Equal(3 + 144, stride);
Assert.Equal(stride * 2, frameBuffer.BufferY!.Width);
Assert.Equal(321, frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, 0, 0, 0)[0]);
Assert.Equal(2, chromaRow.Length);
}
[Fact]
public void Yuv422ToRgbBilinearlyUpsamplesCenteredChroma()
{
@ -333,6 +391,44 @@ public class Av1YuvConverterTests
Assert.Equal(b, actualPixel.B, 2d);
}
[Theory]
[InlineData(Av1BitDepth.TenBit, ObuMatrixCoefficients.Bt709)]
[InlineData(Av1BitDepth.TenBit, ObuMatrixCoefficients.Identity)]
[InlineData(Av1BitDepth.TenBit, ObuMatrixCoefficients.SmpteYCgCo)]
[InlineData(Av1BitDepth.TwelveBit, ObuMatrixCoefficients.Bt709)]
[InlineData(Av1BitDepth.TwelveBit, ObuMatrixCoefficients.Identity)]
[InlineData(Av1BitDepth.TwelveBit, ObuMatrixCoefficients.SmpteYCgCo)]
public void HighBitDepthRoundTrip(int bitDepth, int matrixCoefficients)
{
// Assign
using Image<Rgb24> image = new(3, 1);
Span<Rgb24> source = image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0);
source[0] = new Rgb24(0, 0, 0);
source[1] = new Rgb24(150, 100, 50);
source[2] = new Rgb24(255, 255, 255);
ObuSequenceHeader sequenceHeader = CreateSequenceHeader(
image.Width,
image.Height,
matrixCoefficients: (ObuMatrixCoefficients)matrixCoefficients,
bitDepth: (Av1BitDepth)bitDepth);
using Av1FrameBuffer<byte> frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false);
using Image<Rgb24> actual = new(image.Width, image.Height);
// Act
Av1YuvConverter.ConvertFromRgb(Configuration.Default, image.Frames.RootFrame, frameBuffer);
Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, actual.Frames.RootFrame);
// Assert
Span<Rgb24> actualPixels = actual.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0);
for (int x = 0; x < source.Length; x++)
{
Assert.Equal(source[x].R, actualPixels[x].R, 1D);
Assert.Equal(source[x].G, actualPixels[x].G, 1D);
Assert.Equal(source[x].B, actualPixels[x].B, 1D);
}
}
// [Theory]
// [WithFile(TestImages.Jpeg.Baseline.Winter444_Interleaved, PixelTypes.Rgb24)]
public void RoundTrip(TestImageProvider<Rgb24> provider)
@ -358,7 +454,8 @@ public class Av1YuvConverterTests
bool fullRange = true,
ObuMatrixCoefficients matrixCoefficients = ObuMatrixCoefficients.Bt709,
Av1ColorFormat colorFormat = Av1ColorFormat.Yuv444,
ObuChromoSamplePosition chromaSamplePosition = ObuChromoSamplePosition.Unknown)
ObuChromoSamplePosition chromaSamplePosition = ObuChromoSamplePosition.Unknown,
Av1BitDepth bitDepth = Av1BitDepth.EightBit)
=> new()
{
MaxFrameWidth = width,
@ -366,7 +463,7 @@ public class Av1YuvConverterTests
ColorConfig = new ObuColorConfig
{
IsMonochrome = colorFormat == Av1ColorFormat.Yuv400,
BitDepth = Av1BitDepth.EightBit,
BitDepth = bitDepth,
MatrixCoefficients = matrixCoefficients,
ColorRange = fullRange,
SubSamplingX = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422,

Write Preview
Loading…
Cancel
Save