From 98bad4d1d9a21ea2aa57f021497f9971981db6f5 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Mon, 3 Jun 2024 21:29:29 +0200 Subject: [PATCH] Superblock decoding --- src/ImageSharp/Formats/Heif/Av1/Readme.md | 48 ++++++ .../Heif/Av1/Symbol/Av1SuperblockInfo.cs | 22 ++- .../Formats/Heif/Av1/Symbol/Av1TileDecoder.cs | 144 ++++++++++++++++-- .../Heif/Av1/Symbol/Av1TransformInfo.cs | 8 + 4 files changed, 206 insertions(+), 16 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Readme.md create mode 100644 src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TransformInfo.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Readme.md b/src/ImageSharp/Formats/Heif/Av1/Readme.md new file mode 100644 index 000000000..056d351cb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Readme.md @@ -0,0 +1,48 @@ +# Open Bitstream Unit + +An OBU unit is a unit of parameters encoded in a bitstream format. In AVIF, it contains a single frame. +This frame is coded using no other frame as reference, it is a so called INTRA frame. AV1 movie encoding also defines INTER frames, +which are predictions of one or more other frames. INTER frames are not used in AVIF and therefore this coded ignores INTER frames. + +An OBU section for AVIF consists of the following headers: + +## Temporal delimiter + +In AV1 movies this is a time point. Although irrelevant for AVIF, most implementtions write one such delimiter at the start of the section. + +## Sequence header + +Common herader for a list (or sequence) of frames. For AVIF, this is exaclty 1 frame. For AVIF, this header can be reduced in size when its `ReducedStillPictureHerader` parameter is true. +This setting is recommended, as all the extra parameters are not applicable for AVIF. + +## Frame header + +Can be 3 different OBU types, which define a single INTRA frame in AVIF files. + +## Tile group + +Defines the tiling parameters and contains the parameters its tile using a different coding. + +# Tiling + +In AV1 a frame is made up of 1 or more tiles. The parameters for each tile are entropy encoded using the context aware symbol coding. +These parameters are contained in an OBU tile group header. + +## Superblock + +A tile consists of one or more superblocks. Superblocks can be either 64x64 or 128x128 pixels in size. +This choice is made per frame, and is specified in the `ObuFrameHeader`. +A superblock contains one or more partitions, to further devide the area. + +## Partition + +A superblock contains one or more Partitions. The partition Type determines the number of partitions it is further split in. +Paritions can contain other partitions and blocks. + +## Block + +A block is the smallest are of the image which has the same transformation parameters. A block contains ore or more ModeInfos. + +## ModeInfo + +The smallest unit in the frame. It determines the parameters for an area of 4 by 4 pixels. diff --git a/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1SuperblockInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1SuperblockInfo.cs index fccc6f607..76528351f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1SuperblockInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1SuperblockInfo.cs @@ -5,7 +5,27 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Symbol; internal class Av1SuperblockInfo { - public int[] SuperblockDeltaQ { get; internal set; } = []; + public Av1SuperblockInfo(Av1BlockModeInfo superblockModeInfo, Av1TransformInfo superblockTransformInfo) + { + this.SuperblockModeInfo = superblockModeInfo; + this.SuperblockTransformInfo = superblockTransformInfo; + } + + public int SuperblockDeltaQ { get; internal set; } + + public Av1BlockModeInfo SuperblockModeInfo { get; set; } + + public int[] CoefficientsY { get; set; } = []; + + public int[] CoefficientsU { get; set; } = []; + + public int[] CoefficientsV { get; set; } = []; + + public Av1TransformInfo SuperblockTransformInfo { get; set; } + + public int CdefStrength { get; internal set; } + + public int SuperblockDeltaLoopFilter { get; set; } public Av1BlockModeInfo GetModeInfo(int rowIndex, int columnIndex) => throw new NotImplementedException(); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TileDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TileDecoder.cs index 00812e521..46f682db7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TileDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TileDecoder.cs @@ -14,7 +14,9 @@ internal class Av1TileDecoder : IAv1TileDecoder private static readonly int[] WienerTapsMid = [3, -7, 15]; private const int PartitionProbabilitySet = 4; - private int[] deltaLoopFilter = []; + // Number of Coefficients in a single ModeInfo 4x4 block of pixels (1 DC + 16 AC). + private const int NumberofCoefficients = 1 + 16; + private bool[][][] blockDecoded = []; private int[][] referenceSgrXqd = []; private int[][][] referenceLrWiener = []; @@ -30,12 +32,50 @@ internal class Av1TileDecoder : IAv1TileDecoder private int maxLumaHeight; private int deltaLoopFilterResolution = -1; private int deltaQuantizerResolution = -1; + private int[] coefficientsY = []; + private int[] coefficientsU = []; + private int[] coefficientsV = []; + private int numModeInfosInSuperblock; + private int superblockColumnCount; + private int superblockRowCount; + private Av1SuperblockInfo[] superblockInfos; + private Av1BlockModeInfo[] modeInfos; + private Av1TransformInfo[] transformInfosY; + private Av1TransformInfo[] transformInfosUv; + private int[] deltaQ; + private int[] cdefStrength; + private int[] deltaLoopFilter; public Av1TileDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileInfo tileInfo) { this.FrameInfo = frameInfo; this.SequenceHeader = sequenceHeader; this.TileInfo = tileInfo; + + // init_main_frame_ctxt + int superblockSizeLog2 = this.SequenceHeader.SuperBlockSizeLog2; + int superblockAlignedWidth = Av1Math.AlignPowerOf2(this.SequenceHeader.MaxFrameWidth, superblockSizeLog2); + int superblockAlignedHeight = Av1Math.AlignPowerOf2(this.SequenceHeader.MaxFrameHeight, superblockSizeLog2); + this.superblockColumnCount = superblockAlignedWidth >> superblockSizeLog2; + this.superblockRowCount = superblockAlignedHeight >> superblockSizeLog2; + int superblockCount = this.superblockColumnCount * this.superblockRowCount; + this.numModeInfosInSuperblock = (1 << (superblockSizeLog2 - ObuConstants.ModeInfoSizeLog2)) * (1 << (superblockSizeLog2 - ObuConstants.ModeInfoSizeLog2)); + + this.superblockInfos = new Av1SuperblockInfo[superblockCount]; + this.modeInfos = new Av1BlockModeInfo[superblockCount * this.numModeInfosInSuperblock]; + this.transformInfosY = new Av1TransformInfo[superblockCount * this.numModeInfosInSuperblock]; + this.transformInfosUv = new Av1TransformInfo[2 * superblockCount * this.numModeInfosInSuperblock]; + this.coefficientsY = new int[superblockCount * this.numModeInfosInSuperblock * NumberofCoefficients]; + int subsamplingFactor = (this.SequenceHeader.ColorConfig.SubSamplingX && this.SequenceHeader.ColorConfig.SubSamplingY) ? 2 : + (this.SequenceHeader.ColorConfig.SubSamplingX && !this.SequenceHeader.ColorConfig.SubSamplingY) ? 1 : + (!this.SequenceHeader.ColorConfig.SubSamplingX && !this.SequenceHeader.ColorConfig.SubSamplingY) ? 0 : -1; + Guard.IsFalse(subsamplingFactor == -1, nameof(subsamplingFactor), "Invalid combination of subsampling."); + this.coefficientsU = new int[(superblockCount * this.numModeInfosInSuperblock * NumberofCoefficients) >> subsamplingFactor]; + this.coefficientsV = new int[(superblockCount * this.numModeInfosInSuperblock * NumberofCoefficients) >> subsamplingFactor]; + this.deltaQ = new int[superblockCount]; + this.cdefStrength = new int[superblockCount * (this.SequenceHeader.Use128x128SuperBlock ? 4 : 1)]; + Array.Fill(this.cdefStrength, -1); + this.deltaLoopFilter = new int[superblockCount * ObuConstants.FrameLoopFilterCount]; } public bool SequenceHeaderDone { get; set; } @@ -58,7 +98,7 @@ internal class Av1TileDecoder : IAv1TileDecoder this.aboveContext.Clear(this.TileInfo.TileColumnStartModeInfo[tileColumnIndex], this.TileInfo.TileColumnStartModeInfo[tileColumnIndex - 1]); this.ClearLoopFilterDelta(); - int planesCount = this.SequenceHeader.ColorConfig.ChannelCount; + int planesCount = this.SequenceHeader.ColorConfig.IsMonochrome ? 1 : 3; this.referenceSgrXqd = new int[planesCount][]; this.referenceLrWiener = new int[planesCount][][]; for (int plane = 0; plane < planesCount; plane++) @@ -87,11 +127,24 @@ internal class Av1TileDecoder : IAv1TileDecoder bool readDeltas = this.FrameInfo.DeltaQParameters.IsPresent; + this.ClearBlockDecodedFlags(row, column, superBlock4x4Size); + + int superblockIndex = (superBlockRow * this.superblockColumnCount) + superBlockColumn; + int cdefFactor = this.SequenceHeader.Use128x128SuperBlock ? 4 : 1; + Av1SuperblockInfo superblockInfo = new(this.modeInfos[superblockIndex], this.transformInfosY[superblockIndex]) + { + CoefficientsY = this.coefficientsY, + CoefficientsU = this.coefficientsU, + CoefficientsV = this.coefficientsV, + CdefStrength = this.cdefStrength[superblockIndex * cdefFactor], + SuperblockDeltaLoopFilter = this.deltaLoopFilter[ObuConstants.FrameLoopFilterCount * superblockIndex], + SuperblockDeltaQ = this.deltaQ[superblockIndex] + }; + // Nothing to do for CDEF // this.ClearCdef(row, column); - this.ClearBlockDecodedFlags(row, column, superBlock4x4Size); - this.ReadLoopRestoration(row, column, superBlockSize); - this.ParsePartition(ref reader, row, column, superBlockSize); + // this.ReadLoopRestoration(row, column, superBlockSize); + this.ParsePartition(ref reader, row, column, superBlockSize, superblockInfo); } } } @@ -150,9 +203,8 @@ internal class Av1TileDecoder : IAv1TileDecoder // TODO: Implement } - private void ParsePartition(ref Av1SymbolDecoder reader, int rowIndex, int columnIndex, Av1BlockSize blockSize) + private void ParsePartition(ref Av1SymbolDecoder reader, int rowIndex, int columnIndex, Av1BlockSize blockSize, Av1SuperblockInfo superblockInfo) { - Av1SuperblockInfo superblockInfo = new(); if (rowIndex >= this.TileInfo.TileRowStartModeInfo[rowIndex] || columnIndex >= this.TileInfo.TileColumnStartModeInfo[columnIndex]) { @@ -164,8 +216,8 @@ internal class Av1TileDecoder : IAv1TileDecoder int block4x4Size = blockSize.Get4x4WideCount(); int halfBlock4x4Size = block4x4Size >> 1; int quarterBlock4x4Size = halfBlock4x4Size >> 2; - bool hasRows = rowIndex + halfBlock4x4Size < this.TileInfo.TileRowCount; - bool hasColumns = columnIndex + halfBlock4x4Size < this.TileInfo.TileColumnCount; + bool hasRows = (rowIndex + halfBlock4x4Size) < this.FrameInfo.ModeInfoRowCount; + bool hasColumns = (columnIndex + halfBlock4x4Size) < this.FrameInfo.ModeInfoColumnCount; Av1PartitionType partitionType = Av1PartitionType.Split; if (blockSize < Av1BlockSize.Block8x8) { @@ -194,13 +246,75 @@ internal class Av1TileDecoder : IAv1TileDecoder switch (partitionType) { case Av1PartitionType.Split: - this.ParsePartition(ref reader, rowIndex, columnIndex, subSize); - this.ParsePartition(ref reader, rowIndex, columnIndex + halfBlock4x4Size, subSize); - this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex, subSize); - this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize); + this.ParsePartition(ref reader, rowIndex, columnIndex, subSize, superblockInfo); + this.ParsePartition(ref reader, rowIndex, columnIndex + halfBlock4x4Size, subSize, superblockInfo); + this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex, subSize, superblockInfo); + this.ParsePartition(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize, superblockInfo); break; case Av1PartitionType.None: this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.None); + break; + case Av1PartitionType.Horizontal: + this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.Horizontal); + if (hasRows) + { + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex, subSize, superblockInfo, Av1PartitionType.Horizontal); + } + + break; + case Av1PartitionType.Vertical: + this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.Vertical); + if (hasRows) + { + this.ParseBlock(ref reader, rowIndex, columnIndex + halfBlock4x4Size, subSize, superblockInfo, Av1PartitionType.Vertical); + } + + break; + case Av1PartitionType.HorizontalA: + this.ParseBlock(ref reader, rowIndex, columnIndex, splitSize, superblockInfo, Av1PartitionType.HorizontalA); + this.ParseBlock(ref reader, rowIndex, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.HorizontalA); + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize, superblockInfo, Av1PartitionType.HorizontalA); + break; + case Av1PartitionType.HorizontalB: + this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.HorizontalB); + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex, splitSize, superblockInfo, Av1PartitionType.HorizontalB); + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.HorizontalB); + break; + case Av1PartitionType.VerticalA: + this.ParseBlock(ref reader, rowIndex, columnIndex, splitSize, superblockInfo, Av1PartitionType.VerticalA); + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex, splitSize, superblockInfo, Av1PartitionType.VerticalA); + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, subSize, superblockInfo, Av1PartitionType.VerticalA); + break; + case Av1PartitionType.VerticalB: + this.ParseBlock(ref reader, rowIndex, columnIndex, subSize, superblockInfo, Av1PartitionType.VerticalB); + this.ParseBlock(ref reader, rowIndex, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.VerticalB); + this.ParseBlock(ref reader, rowIndex + halfBlock4x4Size, columnIndex + halfBlock4x4Size, splitSize, superblockInfo, Av1PartitionType.VerticalB); + break; + case Av1PartitionType.Horizontal4: + for (int i = 0; i < 4; i++) + { + int currentBlockRow = rowIndex + (i * quarterBlock4x4Size); + if (i > 0 && currentBlockRow > this.FrameInfo.ModeInfoRowCount) + { + break; + } + + this.ParseBlock(ref reader, currentBlockRow, columnIndex, subSize, superblockInfo, Av1PartitionType.Horizontal4); + } + + break; + case Av1PartitionType.Vertical4: + for (int i = 0; i < 4; i++) + { + int currentBlockColumn = columnIndex + (i * quarterBlock4x4Size); + if (i > 0 && currentBlockColumn > this.FrameInfo.ModeInfoColumnCount) + { + break; + } + + this.ParseBlock(ref reader, rowIndex, currentBlockColumn, subSize, superblockInfo, Av1PartitionType.Vertical4); + } + break; default: throw new NotImplementedException($"Partition type: {partitionType} is not supported."); @@ -334,7 +448,7 @@ internal class Av1TileDecoder : IAv1TileDecoder private void TransformBlock(int plane, int baseX, int baseY, Av1TransformSize transformSize, int x, int y) { - Av1PartitionInfo partitionInfo = new(new(1, Av1BlockSize.Invalid), new(), false, Av1PartitionType.None); + Av1PartitionInfo partitionInfo = new(new(1, Av1BlockSize.Invalid), new(new(1, Av1BlockSize.Invalid), new()), false, Av1PartitionType.None); int startX = (baseX + 4) * x; int startY = (baseY + 4) * y; bool subsamplingX = this.SequenceHeader.ColorConfig.SubSamplingX; @@ -861,7 +975,7 @@ internal class Av1TileDecoder : IAv1TileDecoder bool deltaQuantizerSignBit = reader.ReadLiteral(1) > 0; int reducedDeltaQuantizerIndex = deltaQuantizerSignBit ? -deltaQuantizerAbsolute : deltaQuantizerAbsolute; this.currentQuantizerIndex = Av1Math.Clip3(1, 255, this.currentQuantizerIndex + (reducedDeltaQuantizerIndex << this.deltaQuantizerResolution)); - partitionInfo.SuperblockInfo.SuperblockDeltaQ[0] = this.currentQuantizerIndex; + partitionInfo.SuperblockInfo.SuperblockDeltaQ = this.currentQuantizerIndex; } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TransformInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TransformInfo.cs new file mode 100644 index 000000000..910309f87 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Symbol/Av1TransformInfo.cs @@ -0,0 +1,8 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Symbol; + +internal class Av1TransformInfo +{ +}