Superblock decoding

src/ImageSharp/Formats/Heif/Av1/Readme.md

@@ -0,0 +1,48 @@
+# Open Bitstream Unit
+
+An OBU unit is a unit of parameters encoded in a bitstream format. In AVIF, it contains a single frame.
+This frame is coded using no other frame as reference, it is a so called INTRA frame. AV1 movie encoding also defines INTER frames,
+which are predictions of one or more other frames. INTER frames are not used in AVIF and therefore this coded ignores INTER frames.
+
+An OBU section for AVIF consists of the following headers:
+
+## Temporal delimiter
+
+In AV1 movies this is a time point. Although irrelevant for AVIF, most implementtions write one such delimiter at the start of the section.
+
+## Sequence header
+
+Common herader for a list (or sequence) of frames. For AVIF, this is exaclty 1 frame. For AVIF, this header can be reduced in size when its `ReducedStillPictureHerader` parameter is true. 
+This setting is recommended, as all the extra parameters are not applicable for AVIF.
+
+## Frame header
+
+Can be 3 different OBU types, which define a single INTRA frame in AVIF files.
+
+## Tile group
+
+Defines the tiling parameters and contains the parameters its tile using a different coding.
+
+# Tiling
+
+In AV1 a frame is made up of 1 or more tiles. The parameters for each tile are entropy encoded using the context aware symbol coding.
+These parameters are contained in an OBU tile group header.
+
+## Superblock
+
+A tile consists of one or more superblocks. Superblocks can be either 64x64 or 128x128 pixels in size.
+This choice is made per frame, and is specified in the `ObuFrameHeader`.
+A superblock contains one or more partitions, to further devide the area.
+
+## Partition
+
+A superblock contains one or more Partitions. The partition Type determines the number of partitions it is further split in. 
+Paritions can contain other partitions and blocks.
+
+## Block
+
+A block is the smallest are of the image which has the same transformation parameters. A block contains ore or more ModeInfos.
+
+## ModeInfo
+
+The smallest unit in the frame. It determines the parameters for an area of 4 by 4 pixels.

src/ImageSharp/Formats/Heif/Av1/Symbol/Av1SuperblockInfo.cs

@@ -5,7 +5,27 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Symbol;
 
 internal class Av1SuperblockInfo
 {
-    public int[] SuperblockDeltaQ { get; internal set; } = [];
+    public Av1SuperblockInfo(Av1BlockModeInfo superblockModeInfo, Av1TransformInfo superblockTransformInfo)
+    {
+        this.SuperblockModeInfo = superblockModeInfo;
+        this.SuperblockTransformInfo = superblockTransformInfo;
+    }
+
+    public int SuperblockDeltaQ { get; internal set; }
+
+    public Av1BlockModeInfo SuperblockModeInfo { get; set; }
+
+    public int[] CoefficientsY { get; set; } = [];
+
+    public int[] CoefficientsU { get; set; } = [];
+
+    public int[] CoefficientsV { get; set; } = [];
+
+    public Av1TransformInfo SuperblockTransformInfo { get; set; }
+
+    public int CdefStrength { get; internal set; }
+
+    public int SuperblockDeltaLoopFilter { get; set; }
 
     public Av1BlockModeInfo GetModeInfo(int rowIndex, int columnIndex) => throw new NotImplementedException();
 }

src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TileDecoder.cs

@@ -14,7 +14,9 @@ internal class Av1TileDecoder : IAv1TileDecoder
     private static readonly int[] WienerTapsMid = [3, -7, 15];
     private const int PartitionProbabilitySet = 4;
 
-    private int[] deltaLoopFilter = [];
+    // Number of Coefficients in a single ModeInfo 4x4 block of pixels (1 DC + 16 AC).
+    private const int NumberofCoefficients = 1 + 16;
+
     private bool[][][] blockDecoded = [];
     private int[][] referenceSgrXqd = [];
     private int[][][] referenceLrWiener = [];
@@ -30,12 +32,50 @@ internal class Av1TileDecoder : IAv1TileDecoder
     private int maxLumaHeight;
     private int deltaLoopFilterResolution = -1;
     private int deltaQuantizerResolution = -1;
+    private int[] coefficientsY = [];
+    private int[] coefficientsU = [];
+    private int[] coefficientsV = [];
+    private int numModeInfosInSuperblock;
+    private int superblockColumnCount;
+    private int superblockRowCount;
+    private Av1SuperblockInfo[] superblockInfos;
+    private Av1BlockModeInfo[] modeInfos;
+    private Av1TransformInfo[] transformInfosY;
+    private Av1TransformInfo[] transformInfosUv;
+    private int[] deltaQ;
+    private int[] cdefStrength;
+    private int[] deltaLoopFilter;
 
     public Av1TileDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileInfo tileInfo)
     {
         this.FrameInfo = frameInfo;
         this.SequenceHeader = sequenceHeader;
         this.TileInfo = tileInfo;
+
+        // init_main_frame_ctxt
+        int superblockSizeLog2 = this.SequenceHeader.SuperBlockSizeLog2;
+        int superblockAlignedWidth = Av1Math.AlignPowerOf2(this.SequenceHeader.MaxFrameWidth, superblockSizeLog2);
+        int superblockAlignedHeight = Av1Math.AlignPowerOf2(this.SequenceHeader.MaxFrameHeight, superblockSizeLog2);
+        this.superblockColumnCount = superblockAlignedWidth >> superblockSizeLog2;
+        this.superblockRowCount = superblockAlignedHeight >> superblockSizeLog2;
+        int superblockCount = this.superblockColumnCount * this.superblockRowCount;
+        this.numModeInfosInSuperblock = (1 << (superblockSizeLog2 - ObuConstants.ModeInfoSizeLog2)) * (1 << (superblockSizeLog2 - ObuConstants.ModeInfoSizeLog2));
+
+        this.superblockInfos = new Av1SuperblockInfo[superblockCount];
+        this.modeInfos = new Av1BlockModeInfo[superblockCount * this.numModeInfosInSuperblock];
+        this.transformInfosY = new Av1TransformInfo[superblockCount * this.numModeInfosInSuperblock];
+        this.transformInfosUv = new Av1TransformInfo[2 * superblockCount * this.numModeInfosInSuperblock];
+        this.coefficientsY = new int[superblockCount * this.numModeInfosInSuperblock * NumberofCoefficients];
+        int subsamplingFactor = (this.SequenceHeader.ColorConfig.SubSamplingX && this.SequenceHeader.ColorConfig.SubSamplingY) ? 2 :
+                (this.SequenceHeader.ColorConfig.SubSamplingX && !this.SequenceHeader.ColorConfig.SubSamplingY) ? 1 :
+                (!this.SequenceHeader.ColorConfig.SubSamplingX && !this.SequenceHeader.ColorConfig.SubSamplingY) ? 0 : -1;
+        Guard.IsFalse(subsamplingFactor == -1, nameof(subsamplingFactor), "Invalid combination of subsampling.");
+        this.coefficientsU = new int[(superblockCount * this.numModeInfosInSuperblock * NumberofCoefficients) >> subsamplingFactor];
+        this.coefficientsV = new int[(superblockCount * this.numModeInfosInSuperblock * NumberofCoefficients) >> subsamplingFactor];
+        this.deltaQ = new int[superblockCount];
+        this.cdefStrength = new int[superblockCount * (this.SequenceHeader.Use128x128SuperBlock ? 4 : 1)];
+        Array.Fill(this.cdefStrength, -1);
+        this.deltaLoopFilter = new int[superblockCount * ObuConstants.FrameLoopFilterCount];
     }
 
     public bool SequenceHeaderDone { get; set; }
@@ -58,7 +98,7 @@ internal class Av1TileDecoder : IAv1TileDecoder
 
         this.aboveContext.Clear(this.TileInfo.TileColumnStartModeInfo[tileColumnIndex], this.TileInfo.TileColumnStartModeInfo[tileColumnIndex - 1]);
         this.ClearLoopFilterDelta();
-        int planesCount = this.SequenceHeader.ColorConfig.ChannelCount;
+        int planesCount = this.SequenceHeader.ColorConfig.IsMonochrome ? 1 : 3;
         this.referenceSgrXqd = new int[planesCount][];
         this.referenceLrWiener = new int[planesCount][][];
         for (int plane = 0; plane < planesCount; plane++)
@@ -87,11 +127,24 @@ internal class Av1TileDecoder : IAv1TileDecoder
 
                 bool readDeltas = this.FrameInfo.DeltaQParameters.IsPresent;
 
+                this.ClearBlockDecodedFlags(row, column, superBlock4x4Size);
+
+                int superblockIndex = (superBlockRow * this.superblockColumnCount) + superBlockColumn;
+                int cdefFactor = this.SequenceHeader.Use128x128SuperBlock ? 4 : 1;
+                Av1SuperblockInfo superblockInfo = new(this.modeInfos[superblockIndex], this.transformInfosY[superblockIndex])
+                {
+                    CoefficientsY = this.coefficientsY,
+                    CoefficientsU = this.coefficientsU,
+                    CoefficientsV = this.coefficientsV,
+                    CdefStrength = this.cdefStrength[superblockIndex * cdefFactor],
+                    SuperblockDeltaLoopFilter = this.deltaLoopFilter[ObuConstants.FrameLoopFilterCount * superblockIndex],
+                    SuperblockDeltaQ = this.deltaQ[superblockIndex]
+                };
+
                 // Nothing to do for CDEF
                 // this.ClearCdef(row, column);
-                this.ClearBlockDecodedFlags(row, column, superBlock4x4Size);
-                this.ReadLoopRestoration(row, column, superBlockSize);
-                this.ParsePartition(ref reader, row, column, superBlockSize);
+                // this.ReadLoopRestoration(row, column, superBlockSize);
+                this.ParsePartition(ref reader, row, column, superBlockSize, superblockInfo);
             }
         }
     }
@@ -150,9 +203,8 @@ internal class Av1TileDecoder : IAv1TileDecoder
         // TODO: Implement
     }
 
-    private void ParsePartition(ref Av1SymbolDecoder reader, int rowIndex, int columnIndex, Av1BlockSize blockSize)
+    private void ParsePartition(ref Av1SymbolDecoder reader, int rowIndex, int columnIndex, Av1BlockSize blockSize, Av1SuperblockInfo superblockInfo)
     {
-        Av1SuperblockInfo superblockInfo = new();
         if (rowIndex >= this.TileInfo.TileRowStartModeInfo[rowIndex] ||
             columnIndex >= this.TileInfo.TileColumnStartModeInfo[columnIndex])
         {
@@ -164,8 +216,8 @@ internal class Av1TileDecoder : IAv1TileDecoder
         int block4x4Size = blockSize.Get4x4WideCount();
         int halfBlock4x4Size = block4x4Size >> 1;
         int quarterBlock4x4Size = halfBlock4x4Size >> 2;
-        bool hasRows = rowIndex + halfBlock4x4Size < this.TileInfo.TileRowCount;
-        bool hasColumns = columnIndex + halfBlock4x4Size < this.TileInfo.TileColumnCount;
+        bool hasRows = (rowIndex + halfBlock4x4Size) < this.FrameInfo.ModeInfoRowCount;
+        bool hasColumns = (columnIndex + halfBlock4x4Size) < this.FrameInfo.ModeInfoColumnCount;
         Av1PartitionType partitionType = Av1PartitionType.Split;
         if (blockSize < Av1BlockSize.Block8x8)
         {
@@ -194,13 +246,75 @@ internal class Av1TileDecoder : IAv1TileDecoder
         switch (partitionType)
         {
             case Av1PartitionType.Split:
-                this.ParsePartition(ref reader, rowIndex, columnIndex, subSize);
-                this.ParsePartition(ref reader, rowIndex, columnIndex + halfBlock4x4Size, subSize);
-                this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex, subSize);
-                this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize);
+                this.ParsePartition(ref reader, rowIndex, columnIndex, subSize, superblockInfo);
+                this.ParsePartition(ref reader, rowIndex, columnIndex + halfBlock4x4Size, subSize, superblockInfo);
+                this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex, subSize, superblockInfo);
+                this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize, superblockInfo);
                 break;
             case Av1PartitionType.None:
                 this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.None);
+                break;
+            case Av1PartitionType.Horizontal:
+                this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.Horizontal);
+                if (hasRows)
+                {
+                    this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex, subSize, superblockInfo, Av1PartitionType.Horizontal);
+                }
+
+                break;
+            case Av1PartitionType.Vertical:
+                this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.Vertical);
+                if (hasRows)
+                {
+                    this.ParseBlock(ref reader, rowIndex, columnIndex + halfBlock4x4Size, subSize, superblockInfo, Av1PartitionType.Vertical);
+                }
+
+                break;
+            case Av1PartitionType.HorizontalA:
+                this.ParseBlock(ref reader, rowIndex, columnIndex, splitSize, superblockInfo, Av1PartitionType.HorizontalA);
+                this.ParseBlock(ref reader, rowIndex, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.HorizontalA);
+                this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize, superblockInfo, Av1PartitionType.HorizontalA);
+                break;
+            case Av1PartitionType.HorizontalB:
+                this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.HorizontalB);
+                this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex, splitSize, superblockInfo, Av1PartitionType.HorizontalB);
+                this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.HorizontalB);
+                break;
+            case Av1PartitionType.VerticalA:
+                this.ParseBlock(ref reader, rowIndex, columnIndex, splitSize, superblockInfo, Av1PartitionType.VerticalA);
+                this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex, splitSize, superblockInfo, Av1PartitionType.VerticalA);
+                this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize, superblockInfo, Av1PartitionType.VerticalA);
+                break;
+            case Av1PartitionType.VerticalB:
+                this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.VerticalB);
+                this.ParseBlock(ref reader, rowIndex, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.VerticalB);
+                this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.VerticalB);
+                break;
+            case Av1PartitionType.Horizontal4:
+                for (int i = 0; i < 4; i++)
+                {
+                    int currentBlockRow = rowIndex + (i * quarterBlock4x4Size);
+                    if (i > 0 && currentBlockRow > this.FrameInfo.ModeInfoRowCount)
+                    {
+                        break;
+                    }
+
+                    this.ParseBlock(ref reader, currentBlockRow, columnIndex, subSize, superblockInfo, Av1PartitionType.Horizontal4);
+                }
+
+                break;
+            case Av1PartitionType.Vertical4:
+                for (int i = 0; i < 4; i++)
+                {
+                    int currentBlockColumn = columnIndex + (i * quarterBlock4x4Size);
+                    if (i > 0 && currentBlockColumn > this.FrameInfo.ModeInfoColumnCount)
+                    {
+                        break;
+                    }
+
+                    this.ParseBlock(ref reader, rowIndex, currentBlockColumn, subSize, superblockInfo, Av1PartitionType.Vertical4);
+                }
+
                 break;
             default:
                 throw new NotImplementedException($"Partition type: {partitionType} is not supported.");
@@ -334,7 +448,7 @@ internal class Av1TileDecoder : IAv1TileDecoder
 
     private void TransformBlock(int plane, int baseX, int baseY, Av1TransformSize transformSize, int x, int y)
     {
-        Av1PartitionInfo partitionInfo = new(new(1, Av1BlockSize.Invalid), new(), false, Av1PartitionType.None);
+        Av1PartitionInfo partitionInfo = new(new(1, Av1BlockSize.Invalid), new(new(1, Av1BlockSize.Invalid), new()), false, Av1PartitionType.None);
         int startX = (baseX + 4) * x;
         int startY = (baseY + 4) * y;
         bool subsamplingX = this.SequenceHeader.ColorConfig.SubSamplingX;
@@ -861,7 +975,7 @@ internal class Av1TileDecoder : IAv1TileDecoder
                 bool deltaQuantizerSignBit = reader.ReadLiteral(1) > 0;
                 int reducedDeltaQuantizerIndex = deltaQuantizerSignBit ? -deltaQuantizerAbsolute : deltaQuantizerAbsolute;
                 this.currentQuantizerIndex = Av1Math.Clip3(1, 255, this.currentQuantizerIndex + (reducedDeltaQuantizerIndex << this.deltaQuantizerResolution));
-                partitionInfo.SuperblockInfo.SuperblockDeltaQ[0] = this.currentQuantizerIndex;
+                partitionInfo.SuperblockInfo.SuperblockDeltaQ = this.currentQuantizerIndex;
             }
         }
     }

src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TransformInfo.cs

@@ -0,0 +1,8 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Symbol;
+
+internal class Av1TransformInfo
+{
+}