diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 6f52e238c..7d568ef83 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -58,8 +58,9 @@ This snapshot pins or classifies the available references and failures; it does | --- | --- | --- | --- | | `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. | | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. | -| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | +| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/blockd.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. | +| `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. | | `Av1Inverse2dTransformer` and `Av1InverseTransformerFactory` | AV1 section 7.11.2 inverse transform and reconstruction | libaom `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, and `av1/common/idct.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar transform oracle for coefficient-row traversal, intermediate layout, stage ranges, clipping, and high-bit-depth sample addition. The managed 16-bit overload is also used as a parity oracle for the byte overload. | This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs index acf05bff4..125f029d0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs @@ -177,6 +177,8 @@ internal static class Av1Constants /// public const int QuantificationMatrixLevelCount = 1 << 4; + public const int QuantizationMatrixElementBitCount = 5; + public const int AngleStep = 3; /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs index 29c29b44b..c2de37beb 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs @@ -8,6 +8,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; internal class Av1InverseQuantizationLookup { + // AV1 reuses the adjusted matrix for 64-pixel transform dimensions, while the stored tables omit those duplicate entries. + private static readonly byte[] TransformMatrixIndices = + [ + 0, 1, 2, 3, 3, 4, 5, 6, 7, 8, 9, 3, 3, 10, 11, 12, 13, 8, 9 + ]; + /// /// Gets 16 sets of quantization matrices for chroma and luma and each TX size. /// Matrices for different TX sizes are in fact sub-sampled from the 32x32 and 16x16 sizes, @@ -17,7 +23,7 @@ internal class Av1InverseQuantizationLookup /// Matrices for different QM levels have been rescaled in the frequency domain according /// to different nominal viewing distances. /// - private static int[][][][] InverseWeightTable => + private static readonly int[][][][] InverseWeightTable = [ [ [ @@ -6797,7 +6803,11 @@ internal class Av1InverseQuantizationLookup ]; public static ReadOnlySpan GetQuantizationMatrix(int level, Av1Plane plane, Av1TransformSize transformSize) + { + int[][][] levelMatrices = InverseWeightTable[level]; + int[][] planeMatrices = levelMatrices[Math.Min(1, (int)plane)]; + int transformMatrixIndex = TransformMatrixIndices[(int)transformSize]; - // Transform size must be adjusted. - => InverseWeightTable[level][Math.Min(1, (int)plane)][(int)transformSize]; + return planeMatrices[transformMatrixIndex]; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs index 8aa679dab..94616a802 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs @@ -53,15 +53,18 @@ internal class Av1InverseQuantizer ReadOnlySpan scanIndices = scanOrder.Scan; int maxValue = (1 << (7 + this.sequenceHeader.ColorConfig.BitDepth.GetBitCount())) - 1; int minValue = -(1 << (7 + this.sequenceHeader.ColorConfig.BitDepth.GetBitCount())); - Av1TransformSize qmTransformSize = transformSize.GetAdjusted(); bool usingQuantizationMatrix = this.frameHeader.QuantizationParameters.IsUsingQMatrix; bool lossless = this.frameHeader.LosslessArray[mode.SegmentId]; short dequantDc = this.deQuantsDeltaQ.GetDc(mode.SegmentId, plane); short dequantAc = this.deQuantsDeltaQ.GetAc(mode.SegmentId, plane); - int qmLevel = lossless || !usingQuantizationMatrix ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1 : this.frameHeader.QuantizationParameters.QMatrix[(int)plane]; + int qmLevel = lossless || !usingQuantizationMatrix + ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1 + : this.frameHeader.SegmentationParameters.QMLevel[(int)plane][mode.SegmentId]; + ReadOnlySpan iqMatrix = (transformType.ToClass() == Av1TransformClass.Class2D) ? - Av1InverseQuantizationLookup.GetQuantizationMatrix(qmLevel, plane, qmTransformSize) - : Av1InverseQuantizationLookup.GetQuantizationMatrix(Av1Constants.QuantificationMatrixLevelCount - 1, Av1Plane.Y, qmTransformSize); + Av1InverseQuantizationLookup.GetQuantizationMatrix(qmLevel, plane, transformSize) + : Av1InverseQuantizationLookup.GetQuantizationMatrix(Av1Constants.QuantificationMatrixLevelCount - 1, Av1Plane.Y, transformSize); + int shift = transformSize.GetScale(); int coefficientCount = level[0]; @@ -108,10 +111,10 @@ internal class Av1InverseQuantizer /// private static int GetDeQuantizedValue(short dequant, int coefficientIndex, ReadOnlySpan iqMatrix) { - const int bias = 1 << (Av1ScanOrderConstants.QuantizationMatrixLevelBitCount - 1); + const int bias = 1 << (Av1Constants.QuantizationMatrixElementBitCount - 1); int deQuantifiedValue = dequant; - deQuantifiedValue = ((iqMatrix[coefficientIndex] * deQuantifiedValue) + bias) >> Av1ScanOrderConstants.QuantizationMatrixLevelBitCount; + deQuantifiedValue = ((iqMatrix[coefficientIndex] * deQuantifiedValue) + bias) >> Av1Constants.QuantizationMatrixElementBitCount; return deQuantifiedValue; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs index d752e2009..19728b0d8 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs @@ -105,7 +105,7 @@ internal class Av1PredictionDecoder topNeighbor, leftNeighbor, stride, - mode, + Av1PredictionMode.DC, blockModeInfoColumnOffset, blockModeInfoRowOffset, bitDepth); @@ -253,6 +253,8 @@ internal class Av1PredictionDecoder int transformWidth = transformSize.GetWidth(); int transformHeight = transformSize.GetHeight(); + int transformWidthInModeInfoUnits = transformSize.Get4x4WideCount(); + int transformHeightInModeInfoUnits = transformSize.Get4x4HighCount(); bool usePalette = modeInfo.GetPaletteSize(plane) > 0; @@ -282,8 +284,8 @@ internal class Av1PredictionDecoder // Distance between bottom edge of this pred block to frame bottom edge int yd = (partitionInfo.ModeBlockToBottomEdge >> (3 + subY)) + (partitionInfo.HeightInPixels[(int)plane] - (blockModeInfoRowOffset << Av1Constants.ModeInfoSizeLog2) - transformHeight) - ydOffset; - bool rightAvailable = modeInfoColumn + ((blockModeInfoColumnOffset + transformWidth) << subX) < tileInfo.ModeInfoColumnEnd; - bool bottomAvailable = (yd > 0) && (modeInfoRow + ((blockModeInfoRowOffset + transformHeight) << subY) < tileInfo.ModeInfoRowEnd); + bool rightAvailable = modeInfoColumn + ((blockModeInfoColumnOffset + transformWidthInModeInfoUnits) << subX) < tileInfo.ModeInfoColumnEnd; + bool bottomAvailable = (yd > 0) && (modeInfoRow + ((blockModeInfoRowOffset + transformHeightInModeInfoUnits) << subY) < tileInfo.ModeInfoRowEnd); Av1PartitionType partition = modeInfo.PartitionType; @@ -964,7 +966,7 @@ internal class Av1PredictionDecoder // TODO: Consider creating SIMD version // interpolate half-sample positions - Guard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count)); + DebugGuard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count)); Span input = stackalloc T[MaxUpsampleSize + 3]; T beforeBuffer = Unsafe.Subtract(ref buffer[0], 1); @@ -981,12 +983,15 @@ internal class Av1PredictionDecoder // interpolate half-sample edge positions Unsafe.Subtract(ref buffer[0], 2) = input[0]; + ref T output = ref buffer[0]; for (int i = 0; i < count; i++) { int s = -int.CreateChecked(input[i]) + (9 * int.CreateChecked(input[i + 1])) + (9 * int.CreateChecked(input[i + 2])) - int.CreateChecked(input[i + 3]); s = Av1Math.Clamp((s + 8) >> 4, 0, (1 << bitDepth) - 1); - buffer[(2 * i) - 1] = T.CreateChecked(s); - buffer[2 * i] = input[i + 2]; + + // The AOM edge buffer reserves prefix storage for the samples at indices -2 and -1. + Unsafe.Add(ref output, (2 * i) - 1) = T.CreateChecked(s); + Unsafe.Add(ref output, 2 * i) = input[i + 2]; } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs new file mode 100644 index 000000000..5ac451f61 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs @@ -0,0 +1,34 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +[Trait("Format", "Avif")] +public class Av1InverseQuantizationTests +{ + [Fact] + public void QuantizationMatricesCoverAllLevelsPlanesAndTransformSizes() + { + for (int level = 0; level < Av1Constants.QuantificationMatrixLevelCount; level++) + { + for (Av1Plane plane = Av1Plane.Y; (int)plane < Av1Constants.MaxPlanes; plane++) + { + for (int transformSizeIndex = 0; transformSizeIndex < (int)Av1TransformSize.AllSizes; transformSizeIndex++) + { + Av1TransformSize transformSize = (Av1TransformSize)transformSizeIndex; + Av1TransformSize adjustedSize = transformSize.GetAdjusted(); + int expectedLength = adjustedSize.GetWidth() * adjustedSize.GetHeight(); + + ReadOnlySpan matrix = Av1InverseQuantizationLookup.GetQuantizationMatrix(level, plane, transformSize); + + Assert.Equal(expectedLength, matrix.Length); + } + } + } + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs index 6334a0e04..72f06e896 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs @@ -29,6 +29,7 @@ public class Av1TilingTests } [Theory] + [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC, 18, 16)] [InlineData(TestImages.Heif.Orange4x4, 0x010E, 0x001d, 21, 1)] public void DecodePixelsFirstTile(string filename, int dataOffset, int dataSize, int tileOffset, int superblockCount) {