Decode multi-superblock AV1 frames

HEIF_IMPLEMENTATION_PLAN.md

@@ -58,8 +58,9 @@ This snapshot pins or classifies the available references and failures; it does
 | --- | --- | --- | --- |
 | `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. |
 | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. |
-| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. |
+| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/blockd.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. |
 | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. |
+| `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. |
 | `Av1Inverse2dTransformer` and `Av1InverseTransformerFactory` | AV1 section 7.11.2 inverse transform and reconstruction | libaom `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, and `av1/common/idct.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar transform oracle for coefficient-row traversal, intermediate layout, stage ranges, clipping, and high-bit-depth sample addition. The managed 16-bit overload is also used as a parity oracle for the byte overload. |
 
 This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped.

src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs

@@ -177,6 +177,8 @@ internal static class Av1Constants
     /// </summary>
     public const int QuantificationMatrixLevelCount = 1 << 4;
 
+    public const int QuantizationMatrixElementBitCount = 5;
+
     public const int AngleStep = 3;
 
     /// <summary>

src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs

@@ -8,6 +8,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification;
 
 internal class Av1InverseQuantizationLookup
 {
+    // AV1 reuses the adjusted matrix for 64-pixel transform dimensions, while the stored tables omit those duplicate entries.
+    private static readonly byte[] TransformMatrixIndices =
+    [
+        0, 1, 2, 3, 3, 4, 5, 6, 7, 8, 9, 3, 3, 10, 11, 12, 13, 8, 9
+    ];
+
     /// <summary>
     /// Gets 16 sets of quantization matrices for chroma and luma and each TX size.
     /// Matrices for different TX sizes are in fact sub-sampled from the 32x32 and 16x16 sizes,
@@ -17,7 +23,7 @@ internal class Av1InverseQuantizationLookup
     /// Matrices for different QM levels have been rescaled in the frequency domain according
     /// to different nominal viewing distances.
     /// </summary>
-    private static int[][][][] InverseWeightTable =>
+    private static readonly int[][][][] InverseWeightTable =
     [
         [
             [
@@ -6797,7 +6803,11 @@ internal class Av1InverseQuantizationLookup
     ];
 
     public static ReadOnlySpan<int> GetQuantizationMatrix(int level, Av1Plane plane, Av1TransformSize transformSize)
+    {
+        int[][][] levelMatrices = InverseWeightTable[level];
+        int[][] planeMatrices = levelMatrices[Math.Min(1, (int)plane)];
+        int transformMatrixIndex = TransformMatrixIndices[(int)transformSize];
 
-        // Transform size must be adjusted.
+        return planeMatrices[transformMatrixIndex];
-        => InverseWeightTable[level][Math.Min(1, (int)plane)][(int)transformSize];
+    }
 }

src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs

@@ -53,15 +53,18 @@ internal class Av1InverseQuantizer
         ReadOnlySpan<short> scanIndices = scanOrder.Scan;
         int maxValue = (1 << (7 + this.sequenceHeader.ColorConfig.BitDepth.GetBitCount())) - 1;
         int minValue = -(1 << (7 + this.sequenceHeader.ColorConfig.BitDepth.GetBitCount()));
-        Av1TransformSize qmTransformSize = transformSize.GetAdjusted();
         bool usingQuantizationMatrix = this.frameHeader.QuantizationParameters.IsUsingQMatrix;
         bool lossless = this.frameHeader.LosslessArray[mode.SegmentId];
         short dequantDc = this.deQuantsDeltaQ.GetDc(mode.SegmentId, plane);
         short dequantAc = this.deQuantsDeltaQ.GetAc(mode.SegmentId, plane);
-        int qmLevel = lossless || !usingQuantizationMatrix ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1 : this.frameHeader.QuantizationParameters.QMatrix[(int)plane];
+        int qmLevel = lossless || !usingQuantizationMatrix
+            ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1
+            : this.frameHeader.SegmentationParameters.QMLevel[(int)plane][mode.SegmentId];
+
         ReadOnlySpan<int> iqMatrix = (transformType.ToClass() == Av1TransformClass.Class2D) ?
-            Av1InverseQuantizationLookup.GetQuantizationMatrix(qmLevel, plane, qmTransformSize)
+            Av1InverseQuantizationLookup.GetQuantizationMatrix(qmLevel, plane, transformSize)
-            : Av1InverseQuantizationLookup.GetQuantizationMatrix(Av1Constants.QuantificationMatrixLevelCount - 1, Av1Plane.Y, qmTransformSize);
+            : Av1InverseQuantizationLookup.GetQuantizationMatrix(Av1Constants.QuantificationMatrixLevelCount - 1, Av1Plane.Y, transformSize);
+
         int shift = transformSize.GetScale();
 
         int coefficientCount = level[0];
@@ -108,10 +111,10 @@ internal class Av1InverseQuantizer
     /// </summary>
     private static int GetDeQuantizedValue(short dequant, int coefficientIndex, ReadOnlySpan<int> iqMatrix)
     {
-        const int bias = 1 << (Av1ScanOrderConstants.QuantizationMatrixLevelBitCount - 1);
+        const int bias = 1 << (Av1Constants.QuantizationMatrixElementBitCount - 1);
         int deQuantifiedValue = dequant;
 
-        deQuantifiedValue = ((iqMatrix[coefficientIndex] * deQuantifiedValue) + bias) >> Av1ScanOrderConstants.QuantizationMatrixLevelBitCount;
+        deQuantifiedValue = ((iqMatrix[coefficientIndex] * deQuantifiedValue) + bias) >> Av1Constants.QuantizationMatrixElementBitCount;
         return deQuantifiedValue;
     }
 }

src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs

@@ -105,7 +105,7 @@ internal class Av1PredictionDecoder
                 topNeighbor,
                 leftNeighbor,
                 stride,
-                mode,
+                Av1PredictionMode.DC,
                 blockModeInfoColumnOffset,
                 blockModeInfoRowOffset,
                 bitDepth);
@@ -253,6 +253,8 @@ internal class Av1PredictionDecoder
 
         int transformWidth = transformSize.GetWidth();
         int transformHeight = transformSize.GetHeight();
+        int transformWidthInModeInfoUnits = transformSize.Get4x4WideCount();
+        int transformHeightInModeInfoUnits = transformSize.Get4x4HighCount();
 
         bool usePalette = modeInfo.GetPaletteSize(plane) > 0;
 
@@ -282,8 +284,8 @@ internal class Av1PredictionDecoder
         // Distance between bottom edge of this pred block to frame bottom edge
         int yd = (partitionInfo.ModeBlockToBottomEdge >> (3 + subY)) +
             (partitionInfo.HeightInPixels[(int)plane] - (blockModeInfoRowOffset << Av1Constants.ModeInfoSizeLog2) - transformHeight) - ydOffset;
-        bool rightAvailable = modeInfoColumn + ((blockModeInfoColumnOffset + transformWidth) << subX) < tileInfo.ModeInfoColumnEnd;
+        bool rightAvailable = modeInfoColumn + ((blockModeInfoColumnOffset + transformWidthInModeInfoUnits) << subX) < tileInfo.ModeInfoColumnEnd;
-        bool bottomAvailable = (yd > 0) && (modeInfoRow + ((blockModeInfoRowOffset + transformHeight) << subY) < tileInfo.ModeInfoRowEnd);
+        bool bottomAvailable = (yd > 0) && (modeInfoRow + ((blockModeInfoRowOffset + transformHeightInModeInfoUnits) << subY) < tileInfo.ModeInfoRowEnd);
 
         Av1PartitionType partition = modeInfo.PartitionType;
 
@@ -964,7 +966,7 @@ internal class Av1PredictionDecoder
         // TODO: Consider creating SIMD version
 
         // interpolate half-sample positions
-        Guard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count));
+        DebugGuard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count));
 
         Span<T> input = stackalloc T[MaxUpsampleSize + 3];
         T beforeBuffer = Unsafe.Subtract(ref buffer[0], 1);
@@ -981,12 +983,15 @@ internal class Av1PredictionDecoder
 
         // interpolate half-sample edge positions
         Unsafe.Subtract(ref buffer[0], 2) = input[0];
+        ref T output = ref buffer[0];
         for (int i = 0; i < count; i++)
         {
             int s = -int.CreateChecked(input[i]) + (9 * int.CreateChecked(input[i + 1])) + (9 * int.CreateChecked(input[i + 2])) - int.CreateChecked(input[i + 3]);
             s = Av1Math.Clamp((s + 8) >> 4, 0, (1 << bitDepth) - 1);
-            buffer[(2 * i) - 1] = T.CreateChecked(s);
+
-            buffer[2 * i] = input[i + 2];
+            // The AOM edge buffer reserves prefix storage for the samples at indices -2 and -1.
+            Unsafe.Add(ref output, (2 * i) - 1) = T.CreateChecked(s);
+            Unsafe.Add(ref output, 2 * i) = input[i + 2];
         }
     }
 

tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using SixLabors.ImageSharp.Formats.Heif.Av1;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
+
+[Trait("Format", "Avif")]
+public class Av1InverseQuantizationTests
+{
+    [Fact]
+    public void QuantizationMatricesCoverAllLevelsPlanesAndTransformSizes()
+    {
+        for (int level = 0; level < Av1Constants.QuantificationMatrixLevelCount; level++)
+        {
+            for (Av1Plane plane = Av1Plane.Y; (int)plane < Av1Constants.MaxPlanes; plane++)
+            {
+                for (int transformSizeIndex = 0; transformSizeIndex < (int)Av1TransformSize.AllSizes; transformSizeIndex++)
+                {
+                    Av1TransformSize transformSize = (Av1TransformSize)transformSizeIndex;
+                    Av1TransformSize adjustedSize = transformSize.GetAdjusted();
+                    int expectedLength = adjustedSize.GetWidth() * adjustedSize.GetHeight();
+
+                    ReadOnlySpan<int> matrix = Av1InverseQuantizationLookup.GetQuantizationMatrix(level, plane, transformSize);
+
+                    Assert.Equal(expectedLength, matrix.Length);
+                }
+            }
+        }
+    }
+}

tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs

@@ -29,6 +29,7 @@ public class Av1TilingTests
     }
 
     [Theory]
+    [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC, 18, 16)]
     [InlineData(TestImages.Heif.Orange4x4, 0x010E, 0x001d, 21, 1)]
     public void DecodePixelsFirstTile(string filename, int dataOffset, int dataSize, int tileOffset, int superblockCount)
     {