From a33bc1895abce4647e841d1a1aff525c098e129f Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Thu, 7 Nov 2024 22:21:41 +0100 Subject: [PATCH] Decode superblock immediately after parsing it --- src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs | 8 ++++++- .../Heif/Av1/Pipeline/Av1FrameDecoder.cs | 4 ++-- .../Heif/Av1/Pipeline/IAv1FrameDecoder.cs | 20 ++++++++++++++++ .../Formats/Heif/Av1/Tiling/Av1TileReader.cs | 23 +++++++++++++++---- .../Formats/Heif/Av1/Av1TilingTests.cs | 6 ++++- 5 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index e6ed9ab0fd..5ad394dc06 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -48,7 +48,13 @@ internal class Av1Decoder : IAv1TileReader { this.SequenceHeader = this.obuReader.SequenceHeader; this.FrameHeader = this.obuReader.FrameHeader; - this.tileReader = new Av1TileReader(this.configuration, this.SequenceHeader!, this.FrameHeader!); + Guard.NotNull(this.tileReader, nameof(this.tileReader)); + Guard.NotNull(this.SequenceHeader, nameof(this.SequenceHeader)); + Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader)); + this.FrameInfo = new(this.SequenceHeader); + this.FrameBuffer = new(this.configuration, this.SequenceHeader, this.SequenceHeader.ColorConfig.GetColorFormat(), false); + this.frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, this.FrameBuffer); + this.tileReader = new Av1TileReader(this.configuration, this.SequenceHeader, this.FrameHeader, this.frameDecoder); } this.tileReader.ReadTile(tileData, tileNum); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs index 06946271af..8c1b68d414 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs @@ -8,7 +8,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; -internal class Av1FrameDecoder +internal class Av1FrameDecoder : IAv1FrameDecoder { private readonly ObuSequenceHeader sequenceHeader; private readonly ObuFrameHeader frameHeader; @@ -108,7 +108,7 @@ internal class Av1FrameDecoder /// /// SVT: svt_aom_decode_super_block /// - private void DecodeSuperblock(Point modeInfoPosition, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo) + public void DecodeSuperblock(Point modeInfoPosition, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo) { this.blockDecoder.UpdateSuperblock(superblockInfo); this.inverseQuantizer.UpdateDequant(this.deQuants, superblockInfo); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs new file mode 100644 index 0000000000..1204e8bc1b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs @@ -0,0 +1,20 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Interface for decoder of a single frame. +/// +internal interface IAv1FrameDecoder +{ + /// + /// Decode a single superblock. + /// + /// The top left position of the superblock, in mode info units. + /// The superblock to decode + /// The tile in whcih the superblock is positioned. + void DecodeSuperblock(Point modeInfoPosition, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs index 0ce61bc0cc..ae1050a6c9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -35,12 +36,14 @@ internal class Av1TileReader : IAv1TileReader private readonly int[] firstTransformOffset = new int[2]; private readonly int[] coefficientIndex = []; private readonly Configuration configuration; + private readonly IAv1FrameDecoder frameDecoder; - public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, IAv1FrameDecoder frameDecoder) { this.FrameHeader = frameHeader; this.configuration = configuration; this.SequenceHeader = sequenceHeader; + this.frameDecoder = frameDecoder; // init_main_frame_ctxt this.FrameInfo = new(this.SequenceHeader); @@ -56,7 +59,7 @@ internal class Av1TileReader : IAv1TileReader int superblockColumnCount = Av1Math.AlignPowerOf2(sequenceHeader.MaxFrameWidth, sequenceHeader.SuperblockSizeLog2) >> sequenceHeader.SuperblockSizeLog2; int modeInfoWideColumnCount = superblockColumnCount * sequenceHeader.SuperblockModeInfoSize; - modeInfoWideColumnCount = Av1Math.AlignPowerOf2(modeInfoWideColumnCount, sequenceHeader.SuperblockSizeLog2 - 2); + modeInfoWideColumnCount = Av1Math.AlignPowerOf2(modeInfoWideColumnCount, sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2); this.aboveNeighborContext = new Av1ParseAboveNeighbor4x4Context(planesCount, modeInfoWideColumnCount); this.leftNeighborContext = new Av1ParseLeftNeighbor4x4Context(planesCount, sequenceHeader.SuperblockModeInfoSize); this.transformUnitCount = new int[Av1Constants.MaxPlanes][]; @@ -72,6 +75,9 @@ internal class Av1TileReader : IAv1TileReader public Av1FrameInfo FrameInfo { get; } + /// + /// SVT: parse_tile + /// public void ReadTile(Span tileData, int tileNum) { Av1SymbolDecoder reader = new(tileData, this.FrameHeader.QuantizationParameters.BaseQIndex); @@ -103,14 +109,15 @@ internal class Av1TileReader : IAv1TileReader Av1TileInfo tileInfo = new(tileRowIndex, tileColumnIndex, this.FrameHeader); Av1BlockSize superBlockSize = this.SequenceHeader.SuperblockSize; - int superBlock4x4Size = this.SequenceHeader.SuperblockSizeLog2; + int superBlock4x4Size = this.SequenceHeader.SuperblockSize.Get4x4WideCount(); + int superBlockSizeLog2 = this.SequenceHeader.SuperblockSizeLog2; for (int row = modeInfoRowStart; row < modeInfoRowEnd; row += superBlock4x4Size) { - int superBlockRow = (row << Av1Constants.ModeInfoSizeLog2) >> superBlock4x4Size; + int superBlockRow = (row << Av1Constants.ModeInfoSizeLog2) >> superBlockSizeLog2; this.leftNeighborContext.Clear(this.SequenceHeader); for (int column = modeInfoColumnStart; column < modeInfoColumnEnd; column += superBlock4x4Size) { - int superBlockColumn = (column << Av1Constants.ModeInfoSizeLog2) >> superBlock4x4Size; + int superBlockColumn = (column << Av1Constants.ModeInfoSizeLog2) >> superBlockSizeLog2; Point superblockPosition = new(superBlockColumn, superBlockRow); Av1SuperblockInfo superblockInfo = this.FrameInfo.GetSuperblock(superblockPosition); @@ -120,6 +127,9 @@ internal class Av1TileReader : IAv1TileReader this.firstTransformOffset[1] = 0; this.ReadLoopRestoration(modeInfoPosition, superBlockSize); this.ParsePartition(ref reader, modeInfoPosition, superBlockSize, superblockInfo, tileInfo); + + // decoding of the superblock + this.frameDecoder.DecodeSuperblock(modeInfoPosition, superblockInfo, tileInfo); } } } @@ -1863,6 +1873,9 @@ internal class Av1TileReader : IAv1TileReader return xPos && yPos; }*/ + /// + /// SVT: partition_plane_context + /// private int GetPartitionPlaneContext(Point location, Av1BlockSize blockSize, Av1TileInfo tileInfo, Av1SuperblockInfo superblockInfo) { // Maximum partition point is 8x8. Offset the log value occordingly. diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs index 3c5212b7ac..bab5512167 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs @@ -3,6 +3,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -24,7 +25,10 @@ public class Av1TilingTests IAv1TileReader stub = new Av1TileDecoderStub(); ObuReader obuReader = new(); obuReader.ReadAll(ref bitStreamReader, dataSize, stub); - Av1TileReader tileReader = new(Configuration.Default, obuReader.SequenceHeader, obuReader.FrameHeader); + Av1FrameBuffer frameBuffer = new(Configuration.Default, obuReader.SequenceHeader, Av1ColorFormat.Yuv444, false); + Av1FrameInfo frameInfo = new(obuReader.SequenceHeader); + Av1FrameDecoder frameDecoder = new(obuReader.SequenceHeader, obuReader.FrameHeader, frameInfo, frameBuffer); + Av1TileReader tileReader = new(Configuration.Default, obuReader.SequenceHeader, obuReader.FrameHeader, frameDecoder); // Act tileReader.ReadTile(tileSpan, 0);