From c9baaf019b9380e56722642af4f17f7b0dbf0065 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sat, 24 Aug 2024 14:22:17 +0200 Subject: [PATCH] Prediction decoding --- .../Formats/Heif/Av1/Av1Constants.cs | 2 + src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs | 9 +- .../Formats/Heif/Av1/Av1FrameBuffer.cs | 4 + src/ImageSharp/Formats/Heif/Av1/Av1Math.cs | 3 + .../Av1BottomRightTopLeftConstants.cs | 505 ++++++++ .../Heif/Av1/Prediction/Av1NeighborNeed.cs | 15 + .../Av1/Prediction/Av1PredictorFactory.cs | 18 + .../Prediction/Av1PreditionModeExtensions.cs | 40 + .../Av1ChromaFromLumaContext.cs | 120 ++ .../ChromaFromLuma/Av1ChromaFromLumaMath.cs | 26 + .../Heif/Av1/Prediction/PredictionDecoder.cs | 1094 +++++++++++++++++ .../Heif/Av1/Tiling/Av1BlockModeInfo.cs | 2 + .../Heif/Av1/Tiling/Av1FilterIntraMode.cs | 3 +- .../Heif/Av1/Tiling/Av1PartitionInfo.cs | 63 +- .../Formats/Heif/Av1/Tiling/Av1TileReader.cs | 18 +- .../Formats/Heif/Av1/Av1TilingTests.cs | 2 +- 16 files changed, 1902 insertions(+), 22 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1BottomRightTopLeftConstants.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1NeighborNeed.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaContext.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaMath.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/PredictionDecoder.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs index cb55ea1ad0..a31bb137fe 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs @@ -173,4 +173,6 @@ internal static class Av1Constants /// Total number of Quantification Matrices sets stored. /// public const int QuantificationMatrixLevelCount = 4; + + public const int AngleStep = 3; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index ff26b2e5cc..6f99ada289 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -10,10 +10,15 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1; internal class Av1Decoder : IAv1TileReader { private readonly ObuReader obuReader; + private readonly Configuration configuration; private Av1TileReader? tileReader; private Av1FrameDecoder? frameDecoder; - public Av1Decoder() => this.obuReader = new(); + public Av1Decoder(Configuration configuration) + { + this.configuration = configuration; + this.obuReader = new(); + } public ObuFrameHeader? FrameHeader { get; private set; } @@ -40,7 +45,7 @@ internal class Av1Decoder : IAv1TileReader { this.SequenceHeader = this.obuReader.SequenceHeader; this.FrameHeader = this.obuReader.FrameHeader; - this.tileReader = new Av1TileReader(this.SequenceHeader!, this.FrameHeader!); + this.tileReader = new Av1TileReader(this.configuration, this.SequenceHeader!, this.FrameHeader!); } this.tileReader.ReadTile(tileData, tileNum); diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs index a6a0f77954..301804bf65 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs @@ -33,6 +33,8 @@ internal class Av1FrameBuffer : IDisposable int topPadding = DecoderPaddingValue; int bottomPadding = DecoderPaddingValue; + this.StartPosition = new Point(leftPadding, topPadding); + this.Width = this.MaxWidth; this.Height = this.MaxHeight; int strideY = this.MaxWidth + leftPadding + rightPadding; @@ -86,6 +88,8 @@ internal class Av1FrameBuffer : IDisposable this.BitIncrementCr = null; } + public Point StartPosition { get; private set; } + /// /// Gets the Y luma buffer. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs index 936186a3e0..a023529493 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs @@ -152,4 +152,7 @@ internal static class Av1Math internal static int Modulus8(int value) => value & 0x07; internal static int DivideBy8Floor(int value) => value >> 3; + + internal static int RoundPowerOf2Signed(int value, int n) + => (value < 0) ? -RoundPowerOf2(-value, n) : RoundPowerOf2(value, n); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1BottomRightTopLeftConstants.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1BottomRightTopLeftConstants.cs new file mode 100644 index 0000000000..a3a3e0bd8b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1BottomRightTopLeftConstants.cs @@ -0,0 +1,505 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal class Av1BottomRightTopLeftConstants +{ + // Tables to store if the top-right reference pixels are available. The flags + // are represented with bits, packed into 8-bit integers. E.g., for the 32x32 + // blocks in a 128x128 superblock, the index of the "o" block is 10 (in raster + // order), so its flag is stored at the 3rd bit of the 2nd entry in the table, + // i.e. (table[10 / 8] >> (10 % 8)) & 1. + // . . . . + // . . . . + // . . o . + // . . . . + private static readonly byte[] HasTopRight4x4 = [ + 255, 255, 255, 255, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, 127, 127, 127, 127, 85, 85, + 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, 255, 127, 255, 127, 85, 85, 85, 85, 119, 119, 119, 119, + 85, 85, 85, 85, 127, 127, 127, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, 255, 255, + 255, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, 127, 127, 127, 127, 85, 85, 85, 85, + 119, 119, 119, 119, 85, 85, 85, 85, 255, 127, 255, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, + 85, 85, 127, 127, 127, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, + ]; + + private static readonly byte[] HasTopRight4x8 = [ + 255, 255, 255, 255, 119, 119, 119, 119, 127, 127, 127, 127, 119, 119, 119, 119, 255, 127, 255, 127, 119, 119, + 119, 119, 127, 127, 127, 127, 119, 119, 119, 119, 255, 255, 255, 127, 119, 119, 119, 119, 127, 127, 127, 127, + 119, 119, 119, 119, 255, 127, 255, 127, 119, 119, 119, 119, 127, 127, 127, 127, 119, 119, 119, 119, + ]; + + private static readonly byte[] HasTopRight8x4 = [ + 255, 255, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, 127, 127, 0, 0, 85, 85, + 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, 255, 127, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, + 85, 85, 0, 0, 127, 127, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, + ]; + + private static readonly byte[] HasTopRight8x8 = [ + 255, 255, 85, 85, 119, 119, 85, 85, 127, 127, 85, 85, 119, 119, 85, 85, + 255, 127, 85, 85, 119, 119, 85, 85, 127, 127, 85, 85, 119, 119, 85, 85, + ]; + + private static readonly byte[] HasTopRight8x16 = [ + 255, + 255, + 119, + 119, + 127, + 127, + 119, + 119, + 255, + 127, + 119, + 119, + 127, + 127, + 119, + 119, + ]; + + private static readonly byte[] HasTopRight16x8 = [ + 255, + 0, + 85, + 0, + 119, + 0, + 85, + 0, + 127, + 0, + 85, + 0, + 119, + 0, + 85, + 0, + ]; + + private static readonly byte[] HasTopRight16x16 = [ + 255, + 85, + 119, + 85, + 127, + 85, + 119, + 85, + ]; + + private static readonly byte[] HasTopRight16x32 = [255, 119, 127, 119]; + private static readonly byte[] HasTopRight32x16 = [15, 5, 7, 5]; + private static readonly byte[] HasTopRight32x32 = [95, 87]; + private static readonly byte[] HasTopRight32x64 = [127]; + private static readonly byte[] HasTopRight64x32 = [19]; + private static readonly byte[] HasTopRight64x64 = [7]; + private static readonly byte[] HasTopRight64x128 = [3]; + private static readonly byte[] HasTopRight128x64 = [1]; + private static readonly byte[] HasTopRight128x128 = [1]; + private static readonly byte[] HasTopRight4x16 = [ + 255, 255, 255, 255, 127, 127, 127, 127, 255, 127, 255, 127, 127, 127, 127, 127, + 255, 255, 255, 127, 127, 127, 127, 127, 255, 127, 255, 127, 127, 127, 127, 127, + ]; + + private static readonly byte[] HasTopRight16x4 = [ + 255, 0, 0, 0, 85, 0, 0, 0, 119, 0, 0, 0, 85, 0, 0, 0, 127, 0, 0, 0, 85, 0, 0, 0, 119, 0, 0, 0, 85, 0, 0, 0, + ]; + + private static readonly byte[] HasTopRight8x32 = [ + 255, + 255, + 127, + 127, + 255, + 127, + 127, + 127, + ]; + + private static readonly byte[] HasTopRight32x8 = [ + 15, + 0, + 5, + 0, + 7, + 0, + 5, + 0, + ]; + + private static readonly byte[] HasTopRight16x64 = [255, 127]; + private static readonly byte[] HasTopRight64x16 = [3, 1]; + + private static readonly byte[][] HasTopRightTables = [ + + // 4X4 + HasTopRight4x4, + + // 4X8, 8X4, 8X8 + HasTopRight4x8, + HasTopRight8x4, + HasTopRight8x8, + + // 8X16, 16X8, 16X16 + HasTopRight8x16, + HasTopRight16x8, + HasTopRight16x16, + + // 16X32, 32X16, 32X32 + HasTopRight16x32, + HasTopRight32x16, + HasTopRight32x32, + + // 32X64, 64X32, 64X64 + HasTopRight32x64, + HasTopRight64x32, + HasTopRight64x64, + + // 64x128, 128x64, 128x128 + HasTopRight64x128, + HasTopRight128x64, + HasTopRight128x128, + + // 4x16, 16x4, 8x32 + HasTopRight4x16, + HasTopRight16x4, + HasTopRight8x32, + + // 32x8, 16x64, 64x16 + HasTopRight32x8, + HasTopRight16x64, + HasTopRight64x16 + ]; + + private static readonly byte[] HasTopRightVertical8x8 = [ + 255, 255, 0, 0, 119, 119, 0, 0, 127, 127, 0, 0, 119, 119, 0, 0, + 255, 127, 0, 0, 119, 119, 0, 0, 127, 127, 0, 0, 119, 119, 0, 0, + ]; + + private static readonly byte[] HasTopRightVertical16x16 = [ + 255, + 0, + 119, + 0, + 127, + 0, + 119, + 0, + ]; + + private static readonly byte[] HasTopRightVertical32x32 = [15, 7]; + private static readonly byte[] HasTopRightVertical64x64 = [3]; + + // The _vert_* tables are like the ordinary tables above, but describe the + // order we visit square blocks when doing a PARTITION_VERT_A or + // PARTITION_VERT_B. This is the same order as normal except for on the last + // split where we go vertically (TL, BL, TR, BR). We treat the rectangular block + // as a pair of squares, which means that these tables work correctly for both + // mixed vertical partition types. + // + // There are tables for each of the square sizes. Vertical rectangles (like + // BLOCK_16X32) use their respective "non-vert" table + private static readonly byte[]?[] HasTopRightVerticalTables = [ + + // 4X4 + null, + + // 4X8, 8X4, 8X8 + HasTopRight4x8, + null, + HasTopRightVertical8x8, + + // 8X16, 16X8, 16X16 + HasTopRight8x16, + null, + HasTopRightVertical16x16, + + // 16X32, 32X16, 32X32 + HasTopRight16x32, + null, + HasTopRightVertical32x32, + + // 32X64, 64X32, 64X64 + HasTopRight32x64, + null, + HasTopRightVertical64x64, + + // 64x128, 128x64, 128x128 + HasTopRight64x128, + null, + HasTopRight128x128 + ]; + + // Similar to the has_tr_* tables, but store if the bottom-left reference + // pixels are available. + private static readonly byte[] HasBottomLeft4x4 = [ + 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 1, 1, 1, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, + 85, 85, 0, 0, 1, 0, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 1, 1, 1, 84, 85, 85, 85, + 16, 17, 17, 17, 84, 85, 85, 85, 0, 0, 0, 0, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 1, + 1, 1, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 0, 1, 0, 84, 85, 85, 85, 16, 17, 17, 17, + 84, 85, 85, 85, 0, 1, 1, 1, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 0, 0, 0, + ]; + + private static readonly byte[] HasBottomLeft4x8 = [ + 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 1, 0, 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 0, 0, + 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 1, 0, 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 0, 0, + ]; + + private static readonly byte[] HasBottomLeft8x4 = [ + 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, 254, 255, 0, 1, 254, 255, 84, 85, 254, 255, + 16, 17, 254, 255, 84, 85, 254, 255, 0, 0, 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, + 254, 255, 0, 1, 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, 254, 255, 0, 0, + ]; + + private static readonly byte[] HasBottomLeft8x8 = [ + 84, 85, 16, 17, 84, 85, 0, 1, 84, 85, 16, 17, 84, 85, 0, 0, + 84, 85, 16, 17, 84, 85, 0, 1, 84, 85, 16, 17, 84, 85, 0, 0, + ]; + + private static readonly byte[] HasBottomLeft8x16 = [ + 16, + 17, + 0, + 1, + 16, + 17, + 0, + 0, + 16, + 17, + 0, + 1, + 16, + 17, + 0, + 0, + ]; + + private static readonly byte[] HasBottomLeft16x8 = [ + 254, + 84, + 254, + 16, + 254, + 84, + 254, + 0, + 254, + 84, + 254, + 16, + 254, + 84, + 254, + 0, + ]; + + private static readonly byte[] HasBottomLeft16x16 = [ + 84, + 16, + 84, + 0, + 84, + 16, + 84, + 0, + ]; + + private static readonly byte[] HasBottomLeft16x32 = [16, 0, 16, 0]; + private static readonly byte[] HasBottomLeft32x16 = [78, 14, 78, 14]; + private static readonly byte[] HasBottomLeft32x32 = [4, 4]; + private static readonly byte[] HasBottomLeft32x64 = [0]; + private static readonly byte[] HasBottomLeft64x32 = [34]; + private static readonly byte[] HasBottomLeft64x64 = [0]; + private static readonly byte[] HasBottomLeft64x128 = [0]; + private static readonly byte[] HasBottomLeft128x64 = [0]; + private static readonly byte[] HasBottomLeft128x128 = [0]; + private static readonly byte[] HasBottomLeft4x16 = [ + 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, + ]; + + private static readonly byte[] HasBottomLeft16x4 = [ + 254, 254, 254, 84, 254, 254, 254, 16, 254, 254, 254, 84, 254, 254, 254, 0, + 254, 254, 254, 84, 254, 254, 254, 16, 254, 254, 254, 84, 254, 254, 254, 0, + ]; + + private static readonly byte[] HasBottomLeft8x32 = [ + 0, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + ]; + + private static readonly byte[] HasBottomLeft32x8 = [ + 238, + 78, + 238, + 14, + 238, + 78, + 238, + 14, + ]; + + private static readonly byte[] HasBottomLeft16x64 = [0, 0]; + private static readonly byte[] HasBottomLeft64x16 = [42, 42]; + + private static readonly byte[][] HasBottomLeftTables = [ + + // 4X4 + HasBottomLeft4x4, + + // 4X8, 8X4, 8X8 + HasBottomLeft4x8, + HasBottomLeft8x4, + HasBottomLeft8x8, + + // 8X16, 16X8, 16X16 + HasBottomLeft8x16, + HasBottomLeft16x8, + HasBottomLeft16x16, + + // 16X32, 32X16, 32X32 + HasBottomLeft16x32, + HasBottomLeft32x16, + HasBottomLeft32x32, + + // 32X64, 64X32, 64X64 + HasBottomLeft32x64, + HasBottomLeft64x32, + HasBottomLeft64x64, + + // 64x128, 128x64, 128x128 + HasBottomLeft64x128, + HasBottomLeft128x64, + HasBottomLeft128x128, + + // 4x16, 16x4, 8x32 + HasBottomLeft4x16, + HasBottomLeft16x4, + HasBottomLeft8x32, + + // 32x8, 16x64, 64x16 + HasBottomLeft32x8, + HasBottomLeft16x64, + HasBottomLeft64x16 + ]; + + private static readonly byte[] HasBottomLeftVertical8x8 = [ + 254, 255, 16, 17, 254, 255, 0, 1, 254, 255, 16, 17, 254, 255, 0, 0, + 254, 255, 16, 17, 254, 255, 0, 1, 254, 255, 16, 17, 254, 255, 0, 0, + ]; + + private static readonly byte[] HasBottomLeftVertical16x16 = [ + 254, + 16, + 254, + 0, + 254, + 16, + 254, + 0, + ]; + + private static readonly byte[] HasBottomLeftVertical32x32 = [14, 14]; + private static readonly byte[] HasBottomLeftVertical64x64 = [2]; + + // The _vert_* tables are like the ordinary tables above, but describe the + // order we visit square blocks when doing a PARTITION_VERT_A or + // PARTITION_VERT_B. This is the same order as normal except for on the last + // split where we go vertically (TL, BL, TR, BR). We treat the rectangular block + // as a pair of squares, which means that these tables work correctly for both + // mixed vertical partition types. + // + // There are tables for each of the square sizes. Vertical rectangles (like + // BLOCK_16X32) use their respective "non-vert" table + private static readonly byte[]?[] HasBottomLeftVerticalTables = [ + + // 4X4 + null, + + // 4X8, 8X4, 8X8 + HasBottomLeft4x8, + null, + HasBottomLeftVertical8x8, + + // 8X16, 16X8, 16X16 + HasBottomLeft8x16, + null, + HasBottomLeftVertical16x16, + + // 16X32, 32X16, 32X32 + HasBottomLeft16x32, + null, + HasBottomLeftVertical32x32, + + // 32X64, 64X32, 64X64 + HasBottomLeft32x64, + null, + HasBottomLeftVertical64x64, + + // 64x128, 128x64, 128x128 + HasBottomLeft64x128, + null, + HasBottomLeft128x128]; + + public static bool HasTopRight(Av1PartitionType partitionType, Av1BlockSize blockSize, int blockIndex) + { + int index1 = blockIndex / 8; + int index2 = blockIndex % 8; + Span hasBottomLeftTable = GetHasTopRightTable(partitionType, blockSize); + return ((hasBottomLeftTable[index1] >> index2) & 1) > 0; + } + + public static bool HasBottomLeft(Av1PartitionType partitionType, Av1BlockSize blockSize, int blockIndex) + { + int index1 = blockIndex / 8; + int index2 = blockIndex % 8; + Span hasBottomLeftTable = GetHasBottomLeftTable(partitionType, blockSize); + return ((hasBottomLeftTable[index1] >> index2) & 1) > 0; + } + + private static Span GetHasTopRightTable(Av1PartitionType partition, Av1BlockSize blockSize) + { + byte[]? ret; + + // If this is a mixed vertical partition, look up block size in vertical order. + if (partition is Av1PartitionType.VerticalA or Av1PartitionType.VerticalB) + { + DebugGuard.MustBeLessThan((int)blockSize, (int)Av1BlockSize.SizeS, nameof(blockSize)); + ret = HasTopRightVerticalTables[(int)blockSize]; + } + else + { + ret = HasTopRightTables[(int)blockSize]; + } + + DebugGuard.NotNull(ret, nameof(ret)); + return ret; + } + + private static Span GetHasBottomLeftTable(Av1PartitionType partition, Av1BlockSize blockSize) + { + byte[]? ret; + + // If this is a mixed vertical partition, look up block size in vertical order. + if (partition is Av1PartitionType.VerticalA or Av1PartitionType.VerticalB) + { + DebugGuard.MustBeLessThan((int)blockSize, (int)Av1BlockSize.SizeS, nameof(blockSize)); + ret = HasBottomLeftVerticalTables[(int)blockSize]; + } + else + { + ret = HasBottomLeftTables[(int)blockSize]; + } + + DebugGuard.NotNull(ret, nameof(ret)); + return ret; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1NeighborNeed.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1NeighborNeed.cs new file mode 100644 index 0000000000..81408fad31 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1NeighborNeed.cs @@ -0,0 +1,15 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +[Flags] +internal enum Av1NeighborNeed +{ + Nothing = 0, + Left = 2, + Above = 4, + AboveRight = 8, + AboveLeft = 16, + BottomLeft = 32, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs new file mode 100644 index 0000000000..84a0d75d88 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs @@ -0,0 +1,18 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal class Av1PredictorFactory +{ + internal static void DcPredictor(bool v1, bool v2, Av1TransformSize transformSize, ref byte destination, nuint destinationStride, Span aboveRow, Span leftColumn) => throw new NotImplementedException(); + + internal static void DirectionalPredictor(ref byte destination, nuint destinationStride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, bool upsampleAbove, bool upsampleLeft, int angle) => throw new NotImplementedException(); + + internal static void FilterIntraPredictor(ref byte destination, nuint destinationStride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, Av1FilterIntraMode filterIntraMode) => throw new NotImplementedException(); + + internal static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, ref byte destination, nuint destinationStride, Span aboveRow, Span leftColumn) => throw new NotImplementedException(); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PreditionModeExtensions.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PreditionModeExtensions.cs index d3b1df0357..2b2ca3e4f3 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PreditionModeExtensions.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PreditionModeExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; @@ -23,5 +24,44 @@ internal static class Av1PreditionModeExtensions Av1TransformType.AdstAdst, // PAETH ]; + private static readonly Av1NeighborNeed[] NeedsMap = [ + Av1NeighborNeed.Above | Av1NeighborNeed.Left, // DC + Av1NeighborNeed.Above, // V + Av1NeighborNeed.Left, // H + Av1NeighborNeed.Above | Av1NeighborNeed.AboveRight, // D45 + Av1NeighborNeed.Left | Av1NeighborNeed.Above | Av1NeighborNeed.AboveLeft, // D135 + Av1NeighborNeed.Left | Av1NeighborNeed.Above | Av1NeighborNeed.AboveLeft, // D113 + Av1NeighborNeed.Left | Av1NeighborNeed.Above | Av1NeighborNeed.AboveLeft, // D157 + Av1NeighborNeed.Left | Av1NeighborNeed.BottomLeft, // D203 + Av1NeighborNeed.Above | Av1NeighborNeed.AboveRight, // D67 + Av1NeighborNeed.Left | Av1NeighborNeed.Above, // SMOOTH + Av1NeighborNeed.Left | Av1NeighborNeed.Above, // SMOOTH_V + Av1NeighborNeed.Left | Av1NeighborNeed.Above, // SMOOTH_H + Av1NeighborNeed.Left | Av1NeighborNeed.Above | Av1NeighborNeed.AboveLeft, // PAETH + ]; + + private static readonly int[] AngleMap = [ + 0, + 90, + 180, + 45, + 135, + 113, + 157, + 203, + 67, + 0, + 0, + 0, + 0, + ]; + public static Av1TransformType ToTransformType(this Av1PredictionMode mode) => IntraPreditionMode2TransformType[(int)mode]; + + public static bool IsDirectional(this Av1PredictionMode mode) + => mode is >= Av1PredictionMode.Vertical and <= Av1PredictionMode.Directional67Degrees; + + public static Av1NeighborNeed GetNeighborNeed(this Av1PredictionMode mode) => NeedsMap[(int)mode]; + + public static int ToAngle(this Av1PredictionMode mode) => AngleMap[(int)mode]; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaContext.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaContext.cs new file mode 100644 index 0000000000..c8112028db --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaContext.cs @@ -0,0 +1,120 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; + +internal class Av1ChromaFromLumaContext +{ + private const int BufferLine = 32; + + private int bufferHeight; + private int bufferWidth; + private readonly bool subX; + private readonly bool subY; + + public Av1ChromaFromLumaContext(Configuration configuration, ObuColorConfig colorConfig) + { + this.subX = colorConfig.SubSamplingX; + this.subY = colorConfig.SubSamplingY; + this.Q3Buffer = configuration.MemoryAllocator.Allocate2D(new Size(32, 32), AllocationOptions.Clean); + } + + public Buffer2D Q3Buffer { get; private set; } + + public bool AreParametersComputed { get; private set; } + + public void ComputeParameters(Av1TransformSize transformSize) + { + Guard.IsFalse(this.AreParametersComputed, nameof(this.AreParametersComputed), "Do not call cfl_compute_parameters multiple time on the same values."); + this.Pad(transformSize.GetWidth(), transformSize.GetHeight()); + SubtractAverage(ref this.Q3Buffer[0, 0], transformSize); + this.AreParametersComputed = true; + } + + private void Pad(int width, int height) + { + int diff_width = width - this.bufferWidth; + int diff_height = height - this.bufferHeight; + + if (diff_width > 0) + { + int min_height = height - diff_height; + ref short recon_buf_q3 = ref this.Q3Buffer[width - diff_width, 0]; + for (int j = 0; j < min_height; j++) + { + short last_pixel = Unsafe.Subtract(ref recon_buf_q3, 1); + Guard.IsTrue(Unsafe.IsAddressLessThan(ref Unsafe.Add(ref recon_buf_q3, diff_width), ref this.Q3Buffer[BufferLine, BufferLine]), nameof(recon_buf_q3), "Shall stay within bounds."); + for (int i = 0; i < diff_width; i++) + { + Unsafe.Add(ref recon_buf_q3, i) = last_pixel; + } + + recon_buf_q3 += BufferLine; + } + + this.bufferWidth = width; + } + + if (diff_height > 0) + { + ref short recon_buf_q3 = ref this.Q3Buffer[0, height - diff_height]; + for (int j = 0; j < diff_height; j++) + { + ref short last_row_q3 = ref Unsafe.Subtract(ref recon_buf_q3, BufferLine); + Guard.IsTrue(Unsafe.IsAddressLessThan(ref Unsafe.Add(ref recon_buf_q3, diff_width), ref this.Q3Buffer[BufferLine, BufferLine]), nameof(recon_buf_q3), "Shall stay within bounds."); + for (int i = 0; i < width; i++) + { + Unsafe.Add(ref recon_buf_q3, i) = Unsafe.Add(ref last_row_q3, i); + } + + recon_buf_q3 += BufferLine; + } + + this.bufferHeight = height; + } + } + + /************************************************************************************************ + * svt_subtract_average_c + * Calculate the DC value by averaging over all sample. Subtract DC value to get AC values In C + ************************************************************************************************/ + private static void SubtractAverage(ref short pred_buf_q3, Av1TransformSize transformSize) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int roundOffset = (width * height) >> 1; + int pelCountLog2 = transformSize.GetBlockWidthLog2() + transformSize.GetBlockHeightLog2(); + int sum_q3 = 0; + ref short pred_buf = ref pred_buf_q3; + for (int j = 0; j < height; j++) + { + // assert(pred_buf_q3 + tx_width <= cfl->pred_buf_q3 + CFL_BUF_SQUARE); + for (int i = 0; i < width; i++) + { + sum_q3 += Unsafe.Add(ref pred_buf, i); + } + + pred_buf += BufferLine; + } + + int avg_q3 = (sum_q3 + roundOffset) >> pelCountLog2; + + // Loss is never more than 1/2 (in Q3) + // assert(abs((avg_q3 * (1 << num_pel_log2)) - sum_q3) <= 1 << num_pel_log2 >> + // 1); + for (int j = 0; j < height; j++) + { + for (int i = 0; i < width; i++) + { + Unsafe.Add(ref pred_buf_q3, i) -= (short)avg_q3; + } + + pred_buf_q3 += BufferLine; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaMath.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaMath.cs new file mode 100644 index 0000000000..8f3fce0164 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaMath.cs @@ -0,0 +1,26 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; + +internal static class Av1ChromaFromLumaMath +{ + private const int Signs = 3; + private const int AlphabetSizeLog2 = 4; + + public const int SignZero = 0; + public const int SignNegative = 1; + public const int SignPositive = 2; + + public static int SignU(int jointSign) => ((jointSign + 1) * 11) >> 5; + + public static int SignV(int jointSign) => (jointSign + 1) - (Signs * SignU(jointSign)); + + public static int IndexU(int index) => index >> AlphabetSizeLog2; + + public static int IndexV(int index) => index & (AlphabetSizeLog2 - 1); + + public static int ContextU(int jointSign) => jointSign + 1 - Signs; + + public static int ContextV(int jointSign) => (SignV(jointSign) * Signs) + SignU(jointSign) - Signs; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/PredictionDecoder.cs new file mode 100644 index 0000000000..0e7f9fb216 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/PredictionDecoder.cs @@ -0,0 +1,1094 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal class PredictionDecoder +{ + private const int MaxUpsampleSize = 16; + + private readonly ObuSequenceHeader sequenceHeader; + private readonly ObuFrameHeader frameHeader; + private readonly bool is16BitPipeline; + + public PredictionDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, bool is16BitPipeline) + { + this.sequenceHeader = sequenceHeader; + this.frameHeader = frameHeader; + this.is16BitPipeline = is16BitPipeline; + } + + public void DecodeFrame( + Av1PartitionInfo partitionInfo, + Av1Plane plane, + Av1TransformSize transformSize, + Av1TileInfo tileInfo, + Av1FrameBuffer frameBuffer, + Av1BitDepth bitDepth, + int blockModeInfoColumnOffset, + int blockModeInfoRowOffset) + { + Buffer2D? pixelBuffer = null; + switch (plane) + { + case Av1Plane.Y: + pixelBuffer = frameBuffer.BufferY; + break; + case Av1Plane.U: + pixelBuffer = frameBuffer.BufferCb; + break; + case Av1Plane.V: + pixelBuffer = frameBuffer.BufferCr; + break; + default: + break; + } + + if (pixelBuffer == null) + { + return; + } + + int bytesPerPixel = (bitDepth == Av1BitDepth.EightBit && !this.is16BitPipeline) ? 2 : 1; + ref byte pixelRef = ref pixelBuffer[frameBuffer.StartPosition.X, frameBuffer.StartPosition.Y]; + ref byte topNeighbor = ref pixelRef; + ref byte leftNeighbor = ref pixelRef; + int stride = frameBuffer.BufferY!.Width * bytesPerPixel; + topNeighbor = Unsafe.Subtract(ref topNeighbor, stride); + leftNeighbor = Unsafe.Subtract(ref leftNeighbor, 1); + + bool is16BitPipeline = this.is16BitPipeline; + Av1PredictionMode mode = (plane == Av1Plane.Y) ? partitionInfo.ModeInfo.YMode : partitionInfo.ModeInfo.UvMode; + + if (plane != Av1Plane.Y && partitionInfo.ModeInfo.UvMode == Av1PredictionMode.UvChromaFromLuma) + { + this.PredictIntraBlock( + partitionInfo, + plane, + transformSize, + tileInfo, + ref pixelRef, + stride, + ref topNeighbor, + ref leftNeighbor, + stride, + mode, + blockModeInfoColumnOffset, + blockModeInfoRowOffset, + bitDepth); + + this.PredictChromaFromLumaBlock( + partitionInfo, + partitionInfo.ChromaFromLumaContext, + ref pixelBuffer, + stride, + transformSize, + plane); + + return; + } + + this.PredictIntraBlock( + partitionInfo, + plane, + transformSize, + tileInfo, + ref pixelRef, + stride, + ref topNeighbor, + ref leftNeighbor, + stride, + mode, + blockModeInfoColumnOffset, + blockModeInfoRowOffset, + bitDepth); + } + + private void PredictChromaFromLumaBlock(Av1PartitionInfo partitionInfo, Av1ChromaFromLumaContext? chromaFromLumaContext, ref Buffer2D pixelBuffer, int stride, Av1TransformSize transformSize, Av1Plane plane) + { + Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; + bool isChromaFromLumaAllowedFlag = IsChromaFromLumaAllowedWithFrameHeader(partitionInfo, this.sequenceHeader.ColorConfig, this.frameHeader); + DebugGuard.IsTrue(isChromaFromLumaAllowedFlag, "Chroma from Luma should be allowed then computing it."); + + if (chromaFromLumaContext == null) + { + throw new InvalidOperationException("CFL context should have been defined already."); + } + + if (!chromaFromLumaContext.AreParametersComputed) + { + chromaFromLumaContext.ComputeParameters(transformSize); + } + + int alphaQ3 = ChromaFromLumaIndexToAlpha(modeInfo.ChromaFromLumaAlphaIndex, modeInfo.ChromaFromLumaAlphaSign, (Av1Plane)((int)plane - 1)); + + // assert((transformSize.GetHeight() - 1) * CFL_BUF_LINE + transformSize.GetWidth() <= CFL_BUF_SQUARE); + Av1BitDepth bitDepth = this.sequenceHeader.ColorConfig.BitDepth; + if ((bitDepth != Av1BitDepth.EightBit) || this.is16BitPipeline) + { + /* 16 bit pipeline + svt_cfl_predict_hbd( + chromaFromLumaContext->recon_buf_q3, + (uint16_t*)dst, + dst_stride, + (uint16_t*)dst, + dst_stride, + alpha_q3, + cc->bit_depth, + tx_size_wide[tx_size], + tx_size_high[tx_size]); + return;*/ + } + + ChromaFromLumaPredict( + chromaFromLumaContext.Q3Buffer!, + pixelBuffer, + stride, + pixelBuffer, + stride, + alphaQ3, + bitDepth, + transformSize.GetWidth(), + transformSize.GetHeight()); + } + + private static bool IsChromaFromLumaAllowedWithFrameHeader(Av1PartitionInfo partitionInfo, ObuColorConfig colorConfig, ObuFrameHeader frameHeader) + { + Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; + Av1BlockSize blockSize = modeInfo.BlockSize; + DebugGuard.MustBeGreaterThan((int)blockSize, (int)Av1BlockSize.AllSizes, nameof(blockSize)); + if (frameHeader.LosslessArray[modeInfo.SegmentId]) + { + // In lossless, CfL is available when the partition size is equal to the + // transform size. + bool subX = colorConfig.SubSamplingX; + bool subY = colorConfig.SubSamplingY; + Av1BlockSize planeBlockSize = blockSize.GetSubsampled(subX, subY); + return planeBlockSize == Av1BlockSize.Block4x4; + } + + // Spec: CfL is available to luma partitions lesser than or equal to 32x32 + return blockSize.GetWidth() <= 32 && blockSize.GetHeight() <= 32; + } + + private static int ChromaFromLumaIndexToAlpha(int alphaIndex, int jointSign, Av1Plane plane) + { + int alphaSign = (plane == Av1Plane.U) ? Av1ChromaFromLumaMath.SignU(jointSign) : Av1ChromaFromLumaMath.SignV(jointSign); + if (alphaSign == Av1ChromaFromLumaMath.SignZero) + { + return 0; + } + + int absAlphaQ3 = (plane == Av1Plane.U) ? Av1ChromaFromLumaMath.IndexU(alphaIndex) : Av1ChromaFromLumaMath.IndexV(alphaIndex); + return (alphaSign == Av1ChromaFromLumaMath.SignPositive) ? absAlphaQ3 + 1 : -absAlphaQ3 - 1; + } + + private static int GetScaledLumaQ0(int alphaQ3, short predictedQ3) + { + int scaledLumaQ6 = alphaQ3 * predictedQ3; + return Av1Math.RoundPowerOf2Signed(scaledLumaQ6, 6); + } + + private static void ChromaFromLumaPredict(Buffer2D predictedBufferQ3, Buffer2D predictedBuffer, int predictedStride, Buffer2D destinationBuffer, int destinationStride, int alphaQ3, Av1BitDepth bitDepth, int width, int height) + { + // TODO: Make SIMD variant of this method. + int maxPixelValue = (1 << bitDepth.GetBitCount()) - 1; + for (int j = 0; j < height; j++) + { + for (int i = 0; i < width; i++) + { + int alphaQ0 = GetScaledLumaQ0(alphaQ3, predictedBufferQ3[i, j]); + destinationBuffer[i, j] = (byte)Av1Math.Clamp(alphaQ0 + predictedBuffer[i, j], 0, maxPixelValue); + } + } + } + + private void PredictIntraBlock( + Av1PartitionInfo partitionInfo, + Av1Plane plane, + Av1TransformSize transformSize, + Av1TileInfo tileInfo, + ref byte pixelBuffer, + int pixelBufferStride, + ref byte topNeighbor, + ref byte leftNeighbor, + int referenceStride, + Av1PredictionMode mode, + int blockModeInfoColumnOffset, + int blockModeInfoRowOffset, + Av1BitDepth bitDepth) + { + // TODO:are_parameters_computed variable for CFL so that cal part for V plane we can skip, + // once we compute for U plane, this parameter is block level parameter. + ObuColorConfig cc = this.sequenceHeader.ColorConfig; + int subX = plane != Av1Plane.Y ? cc.SubSamplingX ? 1 : 0 : 0; + int subY = plane != Av1Plane.Y ? cc.SubSamplingY ? 1 : 0 : 0; + + Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; + + int transformWidth = transformSize.GetWidth(); + int transformHeight = transformSize.GetHeight(); + + bool usePalette = modeInfo.GetPaletteSize(plane) > 0; + + if (usePalette) + { + return; + } + + Av1FilterIntraMode filterIntraMode = (plane == Av1Plane.Y && modeInfo.FilterIntraModeInfo.UseFilterIntra) + ? modeInfo.FilterIntraModeInfo.Mode : Av1FilterIntraMode.FilterIntraModes; + + int angleDelta = modeInfo.AngleDelta[Math.Min(1, (int)plane)]; + + Av1BlockSize blockSize = modeInfo.BlockSize; + bool haveTop = blockModeInfoRowOffset > 0 || (subY > 0 ? partitionInfo.AvailableAboveForChroma : partitionInfo.AvailableAbove); + bool haveLeft = blockModeInfoColumnOffset > 0 || (subX > 0 ? partitionInfo.AvailableLeftForChroma : partitionInfo.AvailableLeft); + + int modeInfoRow = -partitionInfo.ModeBlockToTopEdge >> (3 + Av1Constants.ModeInfoSizeLog2); + int modeInfoColumn = -partitionInfo.ModeBlockToLeftEdge >> (3 + Av1Constants.ModeInfoSizeLog2); + int xrOffset = 0; + int ydOffset = 0; + + // Distance between right edge of this pred block to frame right edge + int xr = (partitionInfo.ModeBlockToRightEdge >> (3 + subX)) + (partitionInfo.WidthInPixels[(int)plane] - (blockModeInfoColumnOffset << Av1Constants.ModeInfoSizeLog2) - transformWidth) - + xrOffset; + + // Distance between bottom edge of this pred block to frame bottom edge + int yd = (partitionInfo.ModeBlockToBottomEdge >> (3 + subY)) + + (partitionInfo.HeightInPixels[(int)plane] - (blockModeInfoRowOffset << Av1Constants.ModeInfoSizeLog2) - transformHeight) - ydOffset; + bool rightAvailable = modeInfoColumn + ((blockModeInfoColumnOffset + transformWidth) << subX) < tileInfo.ModeInfoColumnEnd; + bool bottomAvailable = (yd > 0) && (modeInfoRow + ((blockModeInfoRowOffset + transformHeight) << subY) < tileInfo.ModeInfoRowEnd); + + Av1PartitionType partition = modeInfo.PartitionType; + + // force 4x4 chroma component block size. + blockSize = ScaleChromaBlockSize(blockSize, subX == 1, subY == 1); + + bool haveTopRight = IntraHasTopRight( + this.sequenceHeader.SuperblockSize, + blockSize, + modeInfoRow, + modeInfoColumn, + haveTop, + rightAvailable, + partition, + transformSize, + blockModeInfoRowOffset, + blockModeInfoColumnOffset, + subX, + subY); + bool haveBottomLeft = IntraHasBottomLeft( + this.sequenceHeader.SuperblockSize, + blockSize, + modeInfoRow, + modeInfoColumn, + bottomAvailable, + haveLeft, + partition, + transformSize, + blockModeInfoRowOffset, + blockModeInfoColumnOffset, + subX, + subY); + + bool disableEdgeFilter = !this.sequenceHeader.EnableIntraEdgeFilter; + + // Calling all other intra predictors except CFL & pallate... + if (bitDepth == Av1BitDepth.EightBit && !this.is16BitPipeline) + { + this.DecodeBuildIntraPredictors( + partitionInfo, + ref topNeighbor, + ref leftNeighbor, + (nuint)referenceStride, + ref pixelBuffer, + (nuint)pixelBufferStride, + mode, + angleDelta, + filterIntraMode, + transformSize, + disableEdgeFilter, + haveTop ? Math.Min(transformWidth, xr + transformWidth) : 0, + haveTopRight ? Math.Min(transformWidth, xr) : 0, + haveLeft ? Math.Min(transformHeight, yd + transformHeight) : 0, + haveBottomLeft ? Math.Min(transformHeight, yd) : 0, + plane); + } + else + { + /* 16bit + decode_build_intra_predictors_high(xd, + (uint16_t*) top_neigh_array, //As per SVT Enc + (uint16_t*) left_neigh_array, + ref_stride,// As per SVT Enc + (uint16_t*) pv_pred_buf, + pred_stride, + mode, + angle_delta, + filter_intra_mode, + tx_size, + disable_edge_filter, + have_top? AOMMIN(transformWidth, xr + transformWidth) : 0, + have_top_right? AOMMIN(transformWidth, xr) : 0, + have_left? AOMMIN(transformHeight, yd + transformHeight) : 0, + have_bottom_left? AOMMIN(transformHeight, yd) : 0, + plane, + bit_depth); + */ + } + } + + private static Av1BlockSize ScaleChromaBlockSize(Av1BlockSize blockSize, bool subX, bool subY) + { + Av1BlockSize bs = blockSize; + switch (blockSize) + { + case Av1BlockSize.Block4x4: + if (subX && subY) + { + bs = Av1BlockSize.Block8x8; + } + else if (subX) + { + bs = Av1BlockSize.Block8x4; + } + else if (subY) + { + bs = Av1BlockSize.Block4x8; + } + + break; + case Av1BlockSize.Block4x8: + if (subX && subY) + { + bs = Av1BlockSize.Block8x8; + } + else if (subX) + { + bs = Av1BlockSize.Block8x8; + } + else if (subY) + { + bs = Av1BlockSize.Block4x8; + } + + break; + case Av1BlockSize.Block8x4: + if (subX && subY) + { + bs = Av1BlockSize.Block8x8; + } + else if (subX) + { + bs = Av1BlockSize.Block8x4; + } + else if (subY) + { + bs = Av1BlockSize.Block8x8; + } + + break; + case Av1BlockSize.Block4x16: + if (subX && subY) + { + bs = Av1BlockSize.Block8x16; + } + else if (subX) + { + bs = Av1BlockSize.Block8x16; + } + else if (subY) + { + bs = Av1BlockSize.Block4x16; + } + + break; + case Av1BlockSize.Block16x4: + if (subX && subY) + { + bs = Av1BlockSize.Block16x8; + } + else if (subX) + { + bs = Av1BlockSize.Block16x4; + } + else if (subY) + { + bs = Av1BlockSize.Block16x8; + } + + break; + default: + break; + } + + return bs; + } + + private static bool IntraHasBottomLeft(Av1BlockSize superblockSize, Av1BlockSize blockSize, int modeInfoRow, int modeInfoColumn, bool bottomAvailable, bool haveLeft, Av1PartitionType partition, Av1TransformSize transformSize, int blockModeInfoRowOffset, int blockModeInfoColumnOffset, int subX, int subY) + { + if (!bottomAvailable || !haveLeft) + { + return false; + } + + // Special case for 128x* blocks, when col_off is half the block width. + // This is needed because 128x* superblocks are divided into 64x* blocks in + // raster order + if (blockSize.GetWidth() > 64 && blockModeInfoColumnOffset > 0) + { + int planeBlockWidthInUnits64 = 64 >> subX; + int columnOffset64 = blockModeInfoColumnOffset % planeBlockWidthInUnits64; + if (columnOffset64 == 0) + { + // We are at the left edge of top-right or bottom-right 64x* block. + int planeBlockHeightInUnits64 = 64 >> subY; + int rowOffset64 = blockModeInfoRowOffset % planeBlockHeightInUnits64; + int planeBlockHeightInUnits = Math.Min(blockSize.Get4x4HighCount() >> subY, planeBlockHeightInUnits64); + + // Check if all bottom-left pixels are in the left 64x* block (which is + // already coded). + return rowOffset64 + transformSize.Get4x4HighCount() < planeBlockHeightInUnits; + } + } + + if (blockModeInfoColumnOffset > 0) + { + // Bottom-left pixels are in the bottom-left block, which is not available. + return false; + } + else + { + int blockHeightInUnits = blockSize.GetHeight() >> Av1TransformSize.Size4x4.GetBlockHeightLog2(); + int planeBlockHeightInUnits = Math.Max(blockHeightInUnits >> subY, 1); + int bottomLeftUnitCount = transformSize.Get4x4HighCount(); + + // All bottom-left pixels are in the left block, which is already available. + if (blockModeInfoRowOffset + bottomLeftUnitCount < planeBlockHeightInUnits) + { + return true; + } + + int blockWidthInModeInfoLog2 = blockSize.Get4x4WidthLog2(); + int blockHeightInModeInfoLog2 = blockSize.Get4x4HeightLog2(); + int superblockModeInfoSize = superblockSize.Get4x4HighCount(); + int blockRowInSuperblock = (modeInfoRow & (superblockModeInfoSize - 1)) >> blockHeightInModeInfoLog2; + int blockColumnInSuperblock = (modeInfoColumn & (superblockModeInfoSize - 1)) >> blockWidthInModeInfoLog2; + + // Leftmost column of superblock: so bottom-left pixels maybe in the left + // and/or bottom-left superblocks. But only the left superblock is + // available, so check if all required pixels fall in that superblock. + if (blockColumnInSuperblock == 0) + { + int blockStartRowOffset = blockRowInSuperblock << (blockHeightInModeInfoLog2 + Av1Constants.ModeInfoSizeLog2 - Av1TransformSize.Size4x4.GetBlockWidthLog2()) >> subY; + int rowOffsetInSuperblock = blockStartRowOffset + blockModeInfoRowOffset; + int superblockHeightInUnits = superblockModeInfoSize >> subY; + return rowOffsetInSuperblock + bottomLeftUnitCount < superblockHeightInUnits; + } + + // Bottom row of superblock (and not the leftmost column): so bottom-left + // pixels fall in the bottom superblock, which is not available yet. + if (((blockRowInSuperblock + 1) << blockHeightInModeInfoLog2) >= superblockModeInfoSize) + { + return false; + } + + // General case (neither leftmost column nor bottom row): check if the + // bottom-left block is coded before the current block. + int thisBlockIndex = ((blockRowInSuperblock + 0) << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2 - blockWidthInModeInfoLog2)) + blockColumnInSuperblock + 0; + return Av1BottomRightTopLeftConstants.HasBottomLeft(partition, blockSize, thisBlockIndex); + } + } + + private static bool IntraHasTopRight(Av1BlockSize superblockSize, Av1BlockSize blockSize, int modeInfoRow, int modeInfoColumn, bool haveTop, bool rightAvailable, Av1PartitionType partition, Av1TransformSize transformSize, int blockModeInfoRowOffset, int blockModeInfoColumnOffset, int subX, int subY) + { + if (!haveTop || !rightAvailable) + { + return false; + } + + int blockWideInUnits = blockSize.GetWidth() >> 2; + int planeBlockWidthInUnits = Math.Max(blockWideInUnits >> subX, 1); + int topRightUnitCount = transformSize.Get4x4WideCount(); + + if (blockModeInfoRowOffset > 0) + { // Just need to check if enough pixels on the right. + if (blockSize.GetWidth() > 64) + { + // Special case: For 128x128 blocks, the transform unit whose + // top-right corner is at the center of the block does in fact have + // pixels available at its top-right corner. + if (blockModeInfoRowOffset == 64 >> subY && + blockModeInfoColumnOffset + topRightUnitCount == 64 >> subX) + { + return true; + } + + int planeBlockWidthInUnits64 = 64 >> subX; + int blockModeInfoColumnOffset64 = blockModeInfoColumnOffset % planeBlockWidthInUnits64; + return blockModeInfoColumnOffset64 + topRightUnitCount < planeBlockWidthInUnits64; + } + + return blockModeInfoColumnOffset + topRightUnitCount < planeBlockWidthInUnits; + } + else + { + // All top-right pixels are in the block above, which is already available. + if (blockModeInfoColumnOffset + topRightUnitCount < planeBlockWidthInUnits) + { + return true; + } + + int blockWidthInModeInfoLog2 = blockSize.Get4x4WidthLog2(); + int blockHeightInModeInfeLog2 = blockSize.Get4x4HeightLog2(); + int superBlockModeInfoSize = superblockSize.Get4x4HighCount(); + int blockRowInSuperblock = (modeInfoRow & (superBlockModeInfoSize - 1)) >> blockHeightInModeInfeLog2; + int blockColumnInSuperBlock = (modeInfoColumn & (superBlockModeInfoSize - 1)) >> blockWidthInModeInfoLog2; + + // Top row of superblock: so top-right pixels are in the top and/or + // top-right superblocks, both of which are already available. + if (blockRowInSuperblock == 0) + { + return true; + } + + // Rightmost column of superblock (and not the top row): so top-right pixels + // fall in the right superblock, which is not available yet. + if (((blockColumnInSuperBlock + 1) << blockWidthInModeInfoLog2) >= superBlockModeInfoSize) + { + return false; + } + + // General case (neither top row nor rightmost column): check if the + // top-right block is coded before the current block. + int thisBlockIndex = ((blockRowInSuperblock + 0) << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2 - blockWidthInModeInfoLog2)) + blockColumnInSuperBlock + 0; + return Av1BottomRightTopLeftConstants.HasTopRight(partition, blockSize, thisBlockIndex); + } + } + + private void DecodeBuildIntraPredictors( + Av1PartitionInfo partitionInfo, + ref byte aboveNeighbor, + ref byte leftNeighbor, + nuint referenceStride, + ref byte destination, + nuint destinationStride, + Av1PredictionMode mode, + int angleDelta, + Av1FilterIntraMode filterIntraMode, + Av1TransformSize transformSize, + bool disableEdgeFilter, + int topPixelCount, + int topRightPixelCount, + int leftPixelCount, + int bottomLeftPixelCount, + Av1Plane plane) + { + Span aboveData = stackalloc byte[(Av1Constants.MaxTransformSize * 2) + 32]; + Span leftData = stackalloc byte[(Av1Constants.MaxTransformSize * 2) + 32]; + Span aboveRow = aboveData[16..]; + Span leftColumn = leftData[16..]; + int transformWidth = transformSize.GetWidth(); + int transformHeight = transformSize.GetHeight(); + bool isDirectionalMode = mode.IsDirectional(); + Av1NeighborNeed need = mode.GetNeighborNeed(); + bool needLeft = (need & Av1NeighborNeed.Left) == Av1NeighborNeed.Left; + bool needAbove = (need & Av1NeighborNeed.Above) == Av1NeighborNeed.Above; + bool needAboveLeft = (need & Av1NeighborNeed.AboveLeft) == Av1NeighborNeed.AboveLeft; + int angle = 0; + bool useFilterIntra = filterIntraMode != Av1FilterIntraMode.FilterIntraModes; + + if (isDirectionalMode) + { + angle = mode.ToAngle() + (angleDelta * Av1Constants.AngleStep); + if (angle <= 90) + { + needAbove = true; + needLeft = false; + needAboveLeft = true; + } + else if (angle < 180) + { + needAbove = true; + needLeft = true; + needAboveLeft = true; + } + else + { + needAbove = false; + needLeft = true; + needAboveLeft = true; + } + } + + if (useFilterIntra) + { + needAbove = true; + needLeft = true; + needAboveLeft = true; + } + + DebugGuard.MustBeGreaterThanOrEqualTo(topPixelCount, 0, nameof(topPixelCount)); + DebugGuard.MustBeGreaterThanOrEqualTo(topRightPixelCount, 0, nameof(topRightPixelCount)); + DebugGuard.MustBeGreaterThanOrEqualTo(leftPixelCount, 0, nameof(leftPixelCount)); + DebugGuard.MustBeGreaterThanOrEqualTo(bottomLeftPixelCount, 0, nameof(bottomLeftPixelCount)); + + if ((!needAbove && leftPixelCount == 0) || (!needLeft && topPixelCount == 0)) + { + byte val; + if (needLeft) + { + val = (byte)((topPixelCount > 0) ? aboveNeighbor : 129); + } + else + { + val = (byte)((leftPixelCount > 0) ? leftNeighbor : 127); + } + + for (int i = 0; i < transformHeight; ++i) + { + Unsafe.InitBlock(ref destination, val, (uint)transformWidth); + destination = ref Unsafe.Add(ref destination, destinationStride); + } + + return; + } + + // NEED_LEFT + if (needLeft) + { + bool needBottom = (need & Av1NeighborNeed.BottomLeft) == Av1NeighborNeed.BottomLeft; + if (useFilterIntra) + { + needBottom = false; + } + + if (isDirectionalMode) + { + needBottom = angle > 180; + } + + uint numLeftPixelsNeeded = (uint)(transformHeight + (needBottom ? transformWidth : 0)); + int i = 0; + if (leftPixelCount > 0) + { + for (; i < leftPixelCount; i++) + { + leftColumn[i] = Unsafe.Add(ref leftNeighbor, i * (int)referenceStride); + } + + if (needBottom && bottomLeftPixelCount > 0) + { + Guard.IsTrue(i == transformHeight, nameof(i), string.Empty); + for (; i < transformHeight + bottomLeftPixelCount; i++) + { + leftColumn[i] = Unsafe.Add(ref leftNeighbor, i * (int)referenceStride); + } + } + + if (i < numLeftPixelsNeeded) + { + Unsafe.InitBlock(ref leftColumn[i], leftColumn[i - 1], numLeftPixelsNeeded - (uint)i); + } + } + else + { + if (topPixelCount > 0) + { + Unsafe.InitBlock(ref leftColumn[0], aboveNeighbor, numLeftPixelsNeeded); + } + else + { + Unsafe.InitBlock(ref leftColumn[0], 129, numLeftPixelsNeeded); + } + } + } + + // NEED_ABOVE + if (needAbove) + { + bool needRight = (need & Av1NeighborNeed.AboveRight) == Av1NeighborNeed.AboveRight; + if (useFilterIntra) + { + needRight = false; + } + + if (isDirectionalMode) + { + needRight = angle < 90; + } + + uint numTopPixelsNeeded = (uint)(transformWidth + (needRight ? transformHeight : 0)); + if (topPixelCount > 0) + { + Unsafe.CopyBlock(ref aboveRow[0], ref aboveNeighbor, (uint)topPixelCount); + int i = topPixelCount; + if (needRight && topPixelCount > 0) + { + Guard.IsTrue(topPixelCount == transformWidth, nameof(topPixelCount), string.Empty); + Unsafe.CopyBlock(ref aboveRow[transformWidth], ref Unsafe.Add(ref aboveNeighbor, transformWidth), (uint)topPixelCount); + i += topPixelCount; + } + + if (i < numTopPixelsNeeded) + { + Unsafe.InitBlock(ref aboveRow[i], aboveRow[i - 1], numTopPixelsNeeded - (uint)i); + } + } + else + { + if (leftPixelCount > 0) + { + Unsafe.InitBlock(ref aboveRow[0], leftNeighbor, numTopPixelsNeeded); + } + else + { + Unsafe.InitBlock(ref aboveRow[0], 127, numTopPixelsNeeded); + } + } + } + + if (needAboveLeft) + { + if (topPixelCount > 0 && leftPixelCount > 0) + { + aboveRow[-1] = Unsafe.Subtract(ref aboveNeighbor, 1); + } + else if (topPixelCount > 0) + { + aboveRow[-1] = aboveNeighbor; + } + else if (leftPixelCount > 0) + { + aboveRow[-1] = leftNeighbor; + } + else + { + aboveRow[-1] = 128; + } + + leftColumn[-1] = aboveRow[-1]; + } + + if (useFilterIntra) + { + Av1PredictorFactory.FilterIntraPredictor(ref destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode); + return; + } + + if (isDirectionalMode) + { + bool upsampleAbove = false; + bool upsampleLeft = false; + if (!disableEdgeFilter) + { + bool needRight = angle < 90; + bool needBottom = angle > 180; + + bool filterType = GetFilterType(partitionInfo, plane); + + if (angle is not 90 and not 180) + { + int ab_le = needAboveLeft ? 1 : 0; + if (needAbove && needLeft && (transformWidth + transformHeight >= 24)) + { + FilterIntraEdgeCorner(aboveRow, leftColumn); + } + + if (needAbove && topPixelCount > 0) + { + int strength = IntraEdgeFilterStrength(transformWidth, transformHeight, angle - 90, filterType); + int pixelCount = topPixelCount + ab_le + (needRight ? transformHeight : 0); + FilterIntraEdge(ref Unsafe.Subtract(ref aboveRow[0], ab_le), pixelCount, strength); + } + + if (needLeft && leftPixelCount > 0) + { + int strength = IntraEdgeFilterStrength(transformHeight, transformWidth, angle - 180, filterType); + int pixelCount = leftPixelCount + ab_le + (needBottom ? transformWidth : 0); + FilterIntraEdge(ref Unsafe.Subtract(ref leftColumn[0], ab_le), pixelCount, strength); + } + } + + upsampleAbove = UseIntraEdgeUpsample(transformWidth, transformHeight, angle - 90, filterType); + if (needAbove && upsampleAbove) + { + int pixelCount = transformWidth + (needRight ? transformHeight : 0); + + UpsampleIntraEdge(aboveRow, pixelCount); + } + + upsampleLeft = UseIntraEdgeUpsample(transformHeight, transformWidth, angle - 180, filterType); + if (needLeft && upsampleLeft) + { + int pixelCount = transformHeight + (needBottom ? transformWidth : 0); + + UpsampleIntraEdge(leftColumn, pixelCount); + } + } + + Av1PredictorFactory.DirectionalPredictor(ref destination, destinationStride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle); + return; + } + + // predict + if (mode == Av1PredictionMode.DC) + { + Av1PredictorFactory.DcPredictor(leftPixelCount > 0, topPixelCount > 0, transformSize, ref destination, destinationStride, aboveRow, leftColumn); + } + else + { + Av1PredictorFactory.GeneralPredictor(mode, transformSize, ref destination, destinationStride, aboveRow, leftColumn); + } + } + + private static void UpsampleIntraEdge(Span buffer, int count) + { + // TODO: Consider creating SIMD version + + // interpolate half-sample positions + Guard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count)); + + Span input = stackalloc byte[MaxUpsampleSize + 3]; + byte beforeBuffer = Unsafe.Subtract(ref buffer[0], 1); + + // copy p[-1..(sz-1)] and extend first and last samples + input[0] = beforeBuffer; + input[1] = beforeBuffer; + for (int i = 0; i < count; i++) + { + input[i + 2] = buffer[i]; + } + + input[count + 2] = buffer[count - 1]; + + // interpolate half-sample edge positions + buffer[-2] = input[0]; + for (int i = 0; i < count; i++) + { + int s = -input[i] + (9 * input[i + 1]) + (9 * input[i + 2]) - input[i + 3]; + s = Av1Math.Clamp((s + 8) >> 4, 0, 255); + buffer[(2 * i) - 1] = (byte)s; + buffer[2 * i] = input[i + 2]; + } + } + + private static bool UseIntraEdgeUpsample(int width, int height, int delta, bool type) + { + int d = Math.Abs(delta); + int widthHeight = width + height; + if (d is <= 0 or >= 40) + { + return false; + } + + return type ? (widthHeight <= 8) : (widthHeight <= 16); + } + + private static void FilterIntraEdge(ref byte buffer, int count, int strength) + { + // TODO: Consider creating SIMD version + if (strength == 0) + { + return; + } + + int[][] kernel = [ + [0, 4, 8, 4, 0], [0, 5, 6, 5, 0], [2, 4, 4, 4, 2] + ]; + int filt = strength - 1; + Span edge = stackalloc byte[129]; + + Unsafe.CopyBlock(ref edge[0], ref buffer, (uint)count); + for (int i = 1; i < count; i++) + { + int s = 0; + for (int j = 0; j < 5; j++) + { + int k = i - 2 + j; + k = (k < 0) ? 0 : k; + k = (k > count - 1) ? count - 1 : k; + s += edge[k] * kernel[filt][j]; + } + + s = (s + 8) >> 4; + Unsafe.Add(ref buffer, i) = (byte)s; + } + } + + private static int IntraEdgeFilterStrength(int width, int height, int delta, bool filterType) + { + int d = Math.Abs(delta); + int strength = 0; + int widthHeight = width + height; + if (!filterType) + { + if (widthHeight <= 8) + { + if (d >= 56) + { + strength = 1; + } + } + else if (widthHeight <= 12) + { + if (d >= 40) + { + strength = 1; + } + } + else if (widthHeight <= 16) + { + if (d >= 40) + { + strength = 1; + } + } + else if (widthHeight <= 24) + { + if (d >= 8) + { + strength = 1; + } + + if (d >= 16) + { + strength = 2; + } + + if (d >= 32) + { + strength = 3; + } + } + else if (widthHeight <= 32) + { + if (d >= 1) + { + strength = 1; + } + + if (d >= 4) + { + strength = 2; + } + + if (d >= 32) + { + strength = 3; + } + } + else + { + if (d >= 1) + { + strength = 3; + } + } + } + else + { + if (widthHeight <= 8) + { + if (d >= 40) + { + strength = 1; + } + + if (d >= 64) + { + strength = 2; + } + } + else if (widthHeight <= 16) + { + if (d >= 20) + { + strength = 1; + } + + if (d >= 48) + { + strength = 2; + } + } + else if (widthHeight <= 24) + { + if (d >= 4) + { + strength = 3; + } + } + else + { + if (d >= 1) + { + strength = 3; + } + } + } + + return strength; + } + + private static void FilterIntraEdgeCorner(Span above, Span left) + { + int[] kernel = [5, 6, 5]; + + ref byte aboveRef = ref above[0]; + ref byte leftRef = ref left[0]; + ref byte abovePreviousRef = ref Unsafe.Subtract(ref aboveRef, 1); + ref byte leftPreviousRef = ref Unsafe.Subtract(ref leftRef, 1); + int s = (leftRef * kernel[0]) + (abovePreviousRef * kernel[1]) + (aboveRef * kernel[2]); + s = (s + 8) >> 4; + abovePreviousRef = (byte)s; + leftPreviousRef = (byte)s; + } + + private static bool GetFilterType(Av1PartitionInfo partitionInfo, Av1Plane plane) + { + Av1BlockModeInfo? above; + Av1BlockModeInfo? left; + if (plane == Av1Plane.Y) + { + above = partitionInfo.AboveModeInfo; + left = partitionInfo.LeftModeInfo; + } + else + { + above = partitionInfo.AboveModeInfoForChroma; + left = partitionInfo.LeftModeInfoForChroma; + } + + bool aboveIsSmooth = (above != null) && IsSmooth(above, plane); + bool leftIsSmooth = (left != null) && IsSmooth(left, plane); + return aboveIsSmooth || leftIsSmooth; + } + + private static bool IsSmooth(Av1BlockModeInfo modeInfo, Av1Plane plane) + { + if (plane == Av1Plane.Y) + { + Av1PredictionMode mode = modeInfo.YMode; + return mode is Av1PredictionMode.Smooth or + Av1PredictionMode.SmoothVertical or + Av1PredictionMode.SmoothHorizontal; + } + else + { + // Inter mode not supported here. + Av1PredictionMode uvMode = modeInfo.UvMode; + return uvMode is Av1PredictionMode.Smooth or + Av1PredictionMode.SmoothVertical or + Av1PredictionMode.SmoothHorizontal; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs index dbbbf4f50d..3f04731506 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs @@ -62,6 +62,8 @@ internal class Av1BlockModeInfo public int[] TransformUnitsCount { get; internal set; } + public int GetPaletteSize(Av1Plane plane) => this.paletteSize[Math.Min(1, (int)plane)]; + public int GetPaletteSize(Av1PlaneType planeType) => this.paletteSize[(int)planeType]; public void SetPaletteSizes(int ySize, int uvSize) => this.paletteSize = [ySize, uvSize]; diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FilterIntraMode.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FilterIntraMode.cs index f00132db33..eaf33a3d8c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FilterIntraMode.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FilterIntraMode.cs @@ -9,5 +9,6 @@ internal enum Av1FilterIntraMode Vertical, Horizontal, Directional157, - Paeth + Paeth, + FilterIntraModes, } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs index 1c706c34e0..30d506f0cc 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs @@ -2,16 +2,12 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; internal class Av1PartitionInfo { - private int modeBlockToLeftEdge; - private int modeBlockToRightEdge; - private int modeBlockToTopEdge; - private int modeBlockToBottomEdge; - public Av1PartitionInfo(Av1BlockModeInfo modeInfo, Av1SuperblockInfo superblockInfo, bool isChroma, Av1PartitionType partitionType) { this.ModeInfo = modeInfo; @@ -20,6 +16,8 @@ internal class Av1PartitionInfo this.Type = partitionType; this.CdefStrength = []; this.ReferenceFrame = [-1, -1]; + this.WidthInPixels = new int[3]; + this.HeightInPixels = new int[3]; } public Av1BlockModeInfo ModeInfo { get; } @@ -67,37 +65,70 @@ internal class Av1PartitionInfo public Av1BlockModeInfo? LeftModeInfo { get; set; } + public Av1BlockModeInfo? AboveModeInfoForChroma { get; set; } + + public Av1BlockModeInfo? LeftModeInfoForChroma { get; set; } + public int[][] CdefStrength { get; set; } public int[] ReferenceFrame { get; set; } - public int ModeBlockToRightEdge => this.modeBlockToRightEdge; + public int ModeBlockToLeftEdge { get; private set; } + + public int ModeBlockToRightEdge { get; private set; } + + public int ModeBlockToTopEdge { get; private set; } - public int ModeBlockToBottomEdge => this.modeBlockToBottomEdge; + public int ModeBlockToBottomEdge { get; private set; } - public void ComputeBoundaryOffsets(ObuFrameHeader frameHeader, Av1TileInfo tileInfo) + public int[] WidthInPixels { get; private set; } + + public int[] HeightInPixels { get; private set; } + + public Av1ChromaFromLumaContext? ChromaFromLumaContext { get; internal set; } + + public void ComputeBoundaryOffsets(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, Av1TileInfo tileInfo) { Av1BlockSize blockSize = this.ModeInfo.BlockSize; int bw4 = blockSize.Get4x4WideCount(); int bh4 = blockSize.Get4x4HighCount(); + int subX = sequenceHeader.ColorConfig.SubSamplingX ? 1 : 0; + int subY = sequenceHeader.ColorConfig.SubSamplingY ? 1 : 0; this.AvailableAbove = this.RowIndex > tileInfo.ModeInfoRowStart; this.AvailableLeft = this.ColumnIndex > tileInfo.ModeInfoColumnStart; this.AvailableAboveForChroma = this.AvailableAbove; this.AvailableLeftForChroma = this.AvailableLeft; + int shift = Av1Constants.ModeInfoSizeLog2 + 3; - this.modeBlockToLeftEdge = -this.ColumnIndex << shift; - this.modeBlockToRightEdge = (frameHeader.ModeInfoColumnCount - bw4 - this.ColumnIndex) << shift; - this.modeBlockToTopEdge = -this.RowIndex << shift; - this.modeBlockToBottomEdge = (frameHeader.ModeInfoRowCount - bh4 - this.RowIndex) << shift; + this.ModeBlockToLeftEdge = -this.ColumnIndex << shift; + this.ModeBlockToRightEdge = (frameHeader.ModeInfoColumnCount - bw4 - this.ColumnIndex) << shift; + this.ModeBlockToTopEdge = -this.RowIndex << shift; + this.ModeBlockToBottomEdge = (frameHeader.ModeInfoRowCount - bh4 - this.RowIndex) << shift; + + // Block Size width & height in pixels. + // For Luma bock + const int modeInfoSize = 1 << Av1Constants.ModeInfoSizeLog2; + this.WidthInPixels[0] = bw4 * modeInfoSize; + this.HeightInPixels[0] = bh4 * modeInfoSize; + + // For U plane chroma bock + this.WidthInPixels[1] = Math.Max(1, bw4 >> subX) * modeInfoSize; + this.HeightInPixels[1] = Math.Max(1, bh4 >> subY) * modeInfoSize; + + // For V plane chroma bock + this.WidthInPixels[2] = Math.Max(1, bw4 >> subX) * modeInfoSize; + this.HeightInPixels[2] = Math.Max(1, bh4 >> subY) * modeInfoSize; + + this.ChromaFromLumaContext = new Av1ChromaFromLumaContext(configuration, sequenceHeader.ColorConfig); } public int GetMaxBlockWide(Av1BlockSize blockSize, bool subX) { int maxBlockWide = blockSize.GetWidth(); - if (this.modeBlockToRightEdge < 0) + if (this.ModeBlockToRightEdge < 0) { int shift = subX ? 4 : 3; - maxBlockWide += this.modeBlockToRightEdge >> shift; + maxBlockWide += this.ModeBlockToRightEdge >> shift; } return maxBlockWide >> 2; @@ -106,10 +137,10 @@ internal class Av1PartitionInfo public int GetMaxBlockHigh(Av1BlockSize blockSize, bool subY) { int maxBlockHigh = blockSize.GetHeight(); - if (this.modeBlockToBottomEdge < 0) + if (this.ModeBlockToBottomEdge < 0) { int shift = subY ? 4 : 3; - maxBlockHigh += this.modeBlockToBottomEdge >> shift; + maxBlockHigh += this.ModeBlockToBottomEdge >> shift; } return maxBlockHigh >> 2; diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs index ae3f4ddc47..48f45eceeb 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs @@ -34,10 +34,12 @@ internal class Av1TileReader : IAv1TileReader private readonly int[][] transformUnitCount; private readonly int[] firstTransformOffset = new int[2]; private readonly int[] coefficientIndex = []; + private readonly Configuration configuration; - public Av1TileReader(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) { this.FrameHeader = frameHeader; + this.configuration = configuration; this.SequenceHeader = sequenceHeader; // init_main_frame_ctxt @@ -284,6 +286,8 @@ internal class Av1TileReader : IAv1TileReader int block4x4Width = blockSize.Get4x4WideCount(); int block4x4Height = blockSize.Get4x4HighCount(); int planesCount = this.SequenceHeader.ColorConfig.PlaneCount; + int subX = this.SequenceHeader.ColorConfig.SubSamplingX ? 1 : 0; + int subY = this.SequenceHeader.ColorConfig.SubSamplingY ? 1 : 0; Point superblockLocation = superblockInfo.Position * this.SequenceHeader.SuperblockModeInfoSize; Point locationInSuperblock = new Point(modeInfoLocation.X - superblockLocation.X, modeInfoLocation.Y - superblockLocation.Y); Av1BlockModeInfo blockModeInfo = new(planesCount, blockSize, locationInSuperblock); @@ -295,7 +299,7 @@ internal class Av1TileReader : IAv1TileReader partitionInfo.ColumnIndex = columnIndex; partitionInfo.RowIndex = rowIndex; superblockInfo.BlockCount++; - partitionInfo.ComputeBoundaryOffsets(this.FrameHeader, tileInfo); + partitionInfo.ComputeBoundaryOffsets(this.configuration, this.SequenceHeader, this.FrameHeader, tileInfo); if (hasChroma) { if (this.SequenceHeader.ColorConfig.SubSamplingY && block4x4Height == 1) @@ -319,6 +323,16 @@ internal class Av1TileReader : IAv1TileReader partitionInfo.LeftModeInfo = superblockInfo.GetModeInfo(new Point(rowIndex, columnIndex - 1)); } + if (partitionInfo.AvailableAboveForChroma) + { + partitionInfo.AboveModeInfoForChroma = superblockInfo.GetModeInfo(new Point(rowIndex & ~subY, columnIndex | subX)); + } + + if (partitionInfo.AvailableLeftForChroma) + { + partitionInfo.LeftModeInfoForChroma = superblockInfo.GetModeInfo(new Point(rowIndex | subY, columnIndex & ~subX)); + } + this.ReadModeInfo(ref reader, partitionInfo); ReadPaletteTokens(ref reader, partitionInfo); this.ReadBlockTransformSize(ref reader, modeInfoLocation, partitionInfo, superblockInfo, tileInfo); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs index 1c76aa7ec0..e1ad345605 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs @@ -24,7 +24,7 @@ public class Av1TilingTests IAv1TileReader stub = new Av1TileDecoderStub(); ObuReader obuReader = new(); obuReader.ReadAll(ref bitStreamReader, dataSize, stub); - Av1TileReader tileReader = new(obuReader.SequenceHeader, obuReader.FrameHeader); + Av1TileReader tileReader = new(Configuration.Default, obuReader.SequenceHeader, obuReader.FrameHeader); // Act tileReader.ReadTile(tileSpan, 0);