diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 5344fcebd..93e833112 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -67,7 +67,7 @@ This snapshot pins or classifies the available references and failures; it does | Managed implementation | Normative behavior | Reviewed implementation reference | Use | | --- | --- | --- | --- | -| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. | +| `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, chroma reconstruction, and chroma downsampling | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for full/limited-range conversion at 8, 10, and 12 bits. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning. Encode covers the same plane layouts, using libavif's actual-edge-count box average for subsampled chroma. Later SIMD paths must preserve this scalar behavior. | | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. | | `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/blockd.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. | @@ -117,7 +117,7 @@ This assessment is based on the current source after the upstream ImageSharp mer - The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements. - Loop restoration, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches. - The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active intra-prediction, inverse-transform, and block-reconstruction path selects native 16-bit samples for 10/12-bit frames and has focused pipeline wiring coverage. Independently encoded high-bit-depth AVIF conformance files are still required; chroma-from-luma storage and the other normative reconstruction stages listed below remain incomplete. -- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output and uses one allocator-backed RGB row. Constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete. +- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output. Encoder conversion covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with allocator-backed RGB rows and libavif-compatible box averaging. Constant-luminance and chromaticity-derived matrices and ICtCp remain incomplete. - The inverse-transform path allocates arrays in a per-transform hot path. - No usable end-to-end AV1 SIMD path was found. The most visible 4x4 forward-transform SIMD call is commented out, while the production prediction, transform, filter, and output paths are predominantly scalar. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs index 1a10640eb..46db25953 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs @@ -146,7 +146,7 @@ internal static class Av1YuvConverter } /// - /// Converts packed pixels to the YUV 4:4:4 planes used by the AV1 encoder. + /// Converts packed pixels to the configured monochrome or YUV planes used by the AV1 encoder. /// /// The source pixel type. /// The configuration used for allocation and pixel conversion. @@ -167,57 +167,114 @@ internal static class Av1YuvConverter out float chromaScale, out float sampleMaximum); - if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444) - { - throw new NotSupportedException("Only AV1 YUV 4:4:4 encoding color conversion is currently supported."); - } - + bool isMonochrome = frameBuffer.ColorFormat == Av1ColorFormat.Yuv400; + int subX = frameBuffer.ColorConfig.SubSamplingX ? 1 : 0; + int subY = frameBuffer.ColorConfig.SubSamplingY ? 1 : 0; Buffer2DRegion yPlane = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0); - Buffer2DRegion uPlane = frameBuffer.DeriveBlockPointer(Av1Plane.U, 0, 0); - Buffer2DRegion vPlane = frameBuffer.DeriveBlockPointer(Av1Plane.V, 0, 0); - using IMemoryOwner rowOwner = configuration.MemoryAllocator.Allocate(image.Width); - Span rgbRow = rowOwner.GetSpan()[..image.Width]; - - for (int y = 0; y < image.Height; y++) + Buffer2DRegion uPlane = isMonochrome ? default : frameBuffer.DeriveBlockPointer(Av1Plane.U, subX, subY); + Buffer2DRegion vPlane = isMonochrome ? default : frameBuffer.DeriveBlockPointer(Av1Plane.V, subX, subY); + int sourceRowsPerIteration = !isMonochrome && subY != 0 ? 2 : 1; + using IMemoryOwner rowOwner = configuration.MemoryAllocator.Allocate(image.Width * sourceRowsPerIteration); + Span rgbRow0 = rowOwner.GetSpan()[..image.Width]; + Span rgbRow1 = sourceRowsPerIteration == 2 + ? rowOwner.GetSpan().Slice(image.Width, image.Width) + : Span.Empty; + + for (int y = 0; y < image.Height; y += sourceRowsPerIteration) { PixelOperations.Instance.ToRgb24( configuration, image.PixelBuffer.DangerousGetRowSpan(y), - rgbRow); + rgbRow0); + + bool hasSecondSourceRow = sourceRowsPerIteration == 2 && y + 1 < image.Height; + if (hasSecondSourceRow) + { + PixelOperations.Instance.ToRgb24( + configuration, + image.PixelBuffer.DangerousGetRowSpan(y + 1), + rgbRow1); + } if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { - ConvertRgbToYuv444Row( - rgbRow, - yPlane.DangerousGetRowSpan(y), - uPlane.DangerousGetRowSpan(y), - vPlane.DangerousGetRowSpan(y), - mode, - kr, - kg, - kb, - lumaBias, - lumaScale, - chromaBias, - chromaScale, - sampleMaximum); + Span yRow0 = yPlane.DangerousGetRowSpan(y); + if (isMonochrome || subX == 0) + { + ConvertRgbToYuvRow( + rgbRow0, + yRow0, + isMonochrome ? Span.Empty : uPlane.DangerousGetRowSpan(y), + isMonochrome ? Span.Empty : vPlane.DangerousGetRowSpan(y), + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } + else + { + ConvertRgbToSubsampledYuvRows( + rgbRow0, + hasSecondSourceRow ? rgbRow1 : ReadOnlySpan.Empty, + yRow0, + hasSecondSourceRow ? yPlane.DangerousGetRowSpan(y + 1) : Span.Empty, + uPlane.DangerousGetRowSpan(y >> subY), + vPlane.DangerousGetRowSpan(y >> subY), + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } } else { - ConvertRgbToYuv444Row( - rgbRow, - frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0), - frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y, 0, 0), - frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y, 0, 0), - mode, - kr, - kg, - kb, - lumaBias, - lumaScale, - chromaBias, - chromaScale, - sampleMaximum); + Span yRow0 = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0); + if (isMonochrome || subX == 0) + { + ConvertRgbToYuvRow( + rgbRow0, + yRow0, + isMonochrome ? Span.Empty : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y, 0, 0), + isMonochrome ? Span.Empty : frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y, 0, 0), + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } + else + { + ConvertRgbToSubsampledYuvRows( + rgbRow0, + hasSecondSourceRow ? rgbRow1 : ReadOnlySpan.Empty, + yRow0, + hasSecondSourceRow ? frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y + 1, 0, 0) : Span.Empty, + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.U, y >> subY, subX, subY), + frameBuffer.GetHighBitDepthRowSpan(Av1Plane.V, y >> subY, subX, subY), + mode, + kr, + kg, + kb, + lumaBias, + lumaScale, + chromaBias, + chromaScale, + sampleMaximum); + } } } } @@ -485,7 +542,7 @@ internal static class Av1YuvConverter } /// - /// Converts one packed RGB row to YUV 4:4:4 using the resolved H.273 conversion state. + /// Converts one packed RGB row to luma and optional full-resolution chroma using the resolved H.273 conversion state. /// /// The encoded sample type. /// The source RGB pixels. @@ -501,7 +558,7 @@ internal static class Av1YuvConverter /// The encoded chroma midpoint. /// The encoded chroma range. /// The largest encoded sample value. - private static void ConvertRgbToYuv444Row( + private static void ConvertRgbToYuvRow( ReadOnlySpan source, Span yDestination, Span uDestination, @@ -519,44 +576,136 @@ internal static class Av1YuvConverter { for (int x = 0; x < source.Length; x++) { - Rgb24 pixel = source[x]; - float r = pixel.R / ByteMaximum; - float g = pixel.G / ByteMaximum; - float b = pixel.B / ByteMaximum; - float y; - float cb; - float cr; - - switch (mode) - { - case ConversionMode.Identity: - y = g; - cb = b; - cr = r; - break; - case ConversionMode.YCgCo: - y = (0.5F * g) + (0.25F * (r + b)); - cb = (0.5F * g) - (0.25F * (r + b)); - cr = 0.5F * (r - b); - break; - default: - y = (kr * r) + (kg * g) + (kb * b); - cb = (b - y) / (2F * (1F - kb)); - cr = (r - y) / (2F * (1F - kr)); - break; - } + ConvertRgbToYuv(source[x], mode, kr, kg, kb, out float y, out float cb, out float cr); yDestination[x] = ToSample((y * lumaScale) + lumaBias, sampleMaximum); - if (mode == ConversionMode.Identity) + if (!uDestination.IsEmpty) { - uDestination[x] = ToSample((cb * lumaScale) + lumaBias, sampleMaximum); - vDestination[x] = ToSample((cr * lumaScale) + lumaBias, sampleMaximum); + if (mode == ConversionMode.Identity) + { + uDestination[x] = ToSample((cb * lumaScale) + lumaBias, sampleMaximum); + vDestination[x] = ToSample((cr * lumaScale) + lumaBias, sampleMaximum); + } + else + { + uDestination[x] = ToSample((cb * chromaScale) + chromaBias, sampleMaximum); + vDestination[x] = ToSample((cr * chromaScale) + chromaBias, sampleMaximum); + } } - else + } + } + + /// + /// Converts one or two packed RGB rows to luma and horizontally subsampled chroma. + /// + /// The encoded sample type. + /// The first source row. + /// The optional second source row for 4:2:0 conversion. + /// The first destination luma row. + /// The optional second destination luma row. + /// The destination blue-difference chroma row. + /// The destination red-difference chroma row. + /// The conversion mode. + /// The red luma coefficient. + /// The green luma coefficient. + /// The blue luma coefficient. + /// The encoded luma bias. + /// The encoded luma range. + /// The encoded chroma midpoint. + /// The encoded chroma range. + /// The largest encoded sample value. + private static void ConvertRgbToSubsampledYuvRows( + ReadOnlySpan sourceRow0, + ReadOnlySpan sourceRow1, + Span yDestination0, + Span yDestination1, + Span uDestination, + Span vDestination, + ConversionMode mode, + float kr, + float kg, + float kb, + float lumaBias, + float lumaScale, + float chromaBias, + float chromaScale, + float sampleMaximum) + where TSample : unmanaged + { + int rowCount = sourceRow1.IsEmpty ? 1 : 2; + for (int x = 0; x < sourceRow0.Length; x += 2) + { + int columnCount = Math.Min(2, sourceRow0.Length - x); + float cbSum = 0F; + float crSum = 0F; + for (int row = 0; row < rowCount; row++) { - uDestination[x] = ToSample((cb * chromaScale) + chromaBias, sampleMaximum); - vDestination[x] = ToSample((cr * chromaScale) + chromaBias, sampleMaximum); + ReadOnlySpan source = row == 0 ? sourceRow0 : sourceRow1; + Span yDestination = row == 0 ? yDestination0 : yDestination1; + for (int column = 0; column < columnCount; column++) + { + int sourceIndex = x + column; + ConvertRgbToYuv(source[sourceIndex], mode, kr, kg, kb, out float y, out float cb, out float cr); + yDestination[sourceIndex] = ToSample((y * lumaScale) + lumaBias, sampleMaximum); + cbSum += cb; + crSum += cr; + } } + + // libavif's scalar average path divides by the actual edge-block dimensions, so odd widths and heights + // do not replicate a missing RGB sample into the chroma average. + float sampleCount = columnCount * rowCount; + float cbAverage = cbSum / sampleCount; + float crAverage = crSum / sampleCount; + int chromaIndex = x >> 1; + uDestination[chromaIndex] = ToSample((cbAverage * chromaScale) + chromaBias, sampleMaximum); + vDestination[chromaIndex] = ToSample((crAverage * chromaScale) + chromaBias, sampleMaximum); + } + } + + /// + /// Converts one packed RGB pixel to normalized luma and chroma values. + /// + /// The source RGB pixel. + /// The conversion mode. + /// The red luma coefficient. + /// The green luma coefficient. + /// The blue luma coefficient. + /// The normalized luma result. + /// The normalized blue-difference chroma result. + /// The normalized red-difference chroma result. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ConvertRgbToYuv( + Rgb24 pixel, + ConversionMode mode, + float kr, + float kg, + float kb, + out float y, + out float cb, + out float cr) + { + float r = pixel.R / ByteMaximum; + float g = pixel.G / ByteMaximum; + float b = pixel.B / ByteMaximum; + switch (mode) + { + case ConversionMode.Identity: + // H.273 identity coding stores the nonlinear G, B, and R signals in Y, U, and V order. + y = g; + cb = b; + cr = r; + break; + case ConversionMode.YCgCo: + y = (0.5F * g) + (0.25F * (r + b)); + cb = (0.5F * g) - (0.25F * (r + b)); + cr = 0.5F * (r - b); + break; + default: + y = (kr * r) + (kg * g) + (kb * b); + cb = (b - y) / (2F * (1F - kb)); + cr = (r - y) / (2F * (1F - kr)); + break; } }