From 5dedc94d9fdea109b57d9d6b2a5340c6f6845d84 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sun, 7 Jan 2024 21:37:09 +0100 Subject: [PATCH 01/12] Initial av1 bitstream parsing --- .../Formats/Heif/Av1/Av1BitStreamReader.cs | 150 +++ .../Formats/Heif/Av1/Av1BlockSize.cs | 34 + .../Formats/Heif/Av1/Av1ColorFormat.cs | 12 + .../Formats/Heif/Av1/Av1MainParseContext.cs | 8 + src/ImageSharp/Formats/Heif/Av1/Av1Math.cs | 108 +++ src/ImageSharp/Formats/Heif/Av1/Av1Plane.cs | 11 + .../ObuChromoSamplePosition.cs | 22 + .../Av1/OpenBitstreamUnit/ObuColorConfig.cs | 52 + .../OpenBitstreamUnit/ObuColorPrimaries.cs | 21 + .../Av1/OpenBitstreamUnit/ObuConstants.cs | 82 ++ ...tDirectionalEnhancementFilterParameters.cs | 13 + .../ObuFilmGrainParameters.cs | 8 + .../Av1/OpenBitstreamUnit/ObuFrameHeader.cs | 81 ++ .../Av1/OpenBitstreamUnit/ObuFrameSize.cs | 19 + .../Av1/OpenBitstreamUnit/ObuFrameType.cs | 12 + .../Heif/Av1/OpenBitstreamUnit/ObuHeader.cs | 21 + .../ObuLoopFilterParameters.cs | 9 + .../ObuLoopRestorationParameters.cs | 9 + .../ObuMatrixCoefficients.cs | 22 + .../Av1/OpenBitstreamUnit/ObuMetadataType.cs | 13 + .../OpenBitstreamUnit/ObuOperatingPoint.cs | 17 + .../Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs | 13 + .../ObuQuantizationParameters.cs | 19 + .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 888 ++++++++++++++++++ .../Av1/OpenBitstreamUnit/ObuReferenceMode.cs | 10 + .../OpenBitstreamUnit/ObuRestorationType.cs | 9 + .../ObuSegmentationLevelFeature.cs | 16 + .../ObuSegmentationParameters.cs | 15 + .../OpenBitstreamUnit/ObuSequenceHeader.cs | 71 ++ .../OpenBitstreamUnit/ObuSequenceProfile.cs | 11 + .../Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs | 39 + .../ObuTransferCharacteristics.cs | 25 + .../Heif/Av1/OpenBitstreamUnit/ObuType.cs | 18 + 33 files changed, 1858 insertions(+) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1ColorFormat.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1MainParseContext.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1Math.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1Plane.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorConfig.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorPrimaries.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameType.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuHeader.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMatrixCoefficients.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMetadataType.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuQuantizationParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReferenceMode.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuRestorationType.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationLevelFeature.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceProfile.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTransferCharacteristics.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuType.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs new file mode 100644 index 0000000000..e16553121c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs @@ -0,0 +1,150 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal ref struct Av1BitStreamReader(Span data) +{ + private const int WordSize = sizeof(byte); + private const int DoubleWordSize = 2 * WordSize; + private readonly Span data = data; + private int wordPosition = 0; + private int bitOffset = 0; + + public readonly int BitPosition => (this.wordPosition * WordSize) + this.bitOffset; + + public readonly int Length => this.data.Length; + + public void Reset() + { + this.wordPosition = 0; + this.bitOffset = 0; + } + + public void Skip(int bitCount) + { + this.bitOffset += bitCount; + while (this.bitOffset >= WordSize) + { + this.bitOffset -= WordSize; + this.wordPosition++; + } + } + + public uint ReadLiteral(int bitCount) + { + uint bits = (uint)(this.data[this.wordPosition] << this.bitOffset) >> (WordSize - bitCount); + this.bitOffset += bitCount; + while (this.bitOffset > WordSize) + { + uint nextWord = this.data[this.wordPosition + 1]; + bits |= nextWord << (DoubleWordSize - bitCount); + } + + if (this.bitOffset >= WordSize) + { + this.bitOffset -= WordSize; + } + + return bits; + } + + internal bool ReadBoolean() + { + bool bit = (this.data[this.wordPosition] & (1 << (WordSize - this.bitOffset))) > 0; + this.Skip(1); + return bit; + } + + public ulong ReadLittleEndianBytes128(out int length) + { + // See section 4.10.5 of the AV1-Specification + DebugGuard.IsTrue((this.bitOffset & (WordSize - 1)) == 0, "Reading of Little Endian 128 value only allowed on byte alignment"); + + ulong value = 0; + length = 0; + for (int i = 0; i < 56; i += 7) + { + uint leb128Byte = this.ReadLiteral(8); + value |= (leb128Byte & 0x7FUL) << i; + length++; + if ((leb128Byte & 0x80U) != 0x80U) + { + break; + } + } + + return value; + } + + public uint ReadUnsignedVariableLength() + { + // See section 4.10.3 of the AV1-Specification + int leadingZerosCount = 0; + while (leadingZerosCount < 32 && this.ReadLiteral(1) == 0U) + { + leadingZerosCount++; + } + + if (leadingZerosCount == 32) + { + return uint.MaxValue; + } + + uint basis = (1U << leadingZerosCount) - 1U; + uint value = this.ReadLiteral(leadingZerosCount); + return basis + value; + } + + public uint ReadNonSymmetric(uint n) + { + // See section 4.10.7 of the AV1-Specification + if (n <= 1) + { + return 0; + } + + int w = (int)(Av1Math.MostSignificantBit(n) + 1); + uint m = (uint)((1 << w) - n); + uint v = this.ReadLiteral(w - 1); + if (v < m) + { + return v; + } + + return (v << 1) - m + this.ReadLiteral(1); + } + + public int ReadSignedFromUnsigned(int n) + { + // See section 4.10.6 of the AV1-Specification + int signedValue; + uint value = this.ReadLiteral(n); + uint signMask = 1U << (n - 1); + if ((value & signMask) == signMask) + { + // Prevent overflow by casting to long; + signedValue = (int)((long)value - (signMask << 1)); + } + else + { + signedValue = (int)value; + } + + return signedValue; + } + + public uint ReadLittleEndian(int n) + { + // See section 4.10.4 of the AV1-Specification + DebugGuard.IsTrue((this.bitOffset & (WordSize - 1)) == 0, "Reading of Little Endian value only allowed on byte alignment"); + + uint t = 0; + for (int i = 0; i < 8 * n; i += 8) + { + t += this.ReadLiteral(8) << i; + } + + return t; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs new file mode 100644 index 0000000000..6eca5bc638 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs @@ -0,0 +1,34 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal enum Av1BlockSize +{ + Block4x4, + Block4x8, + Block8x4, + Block8x8, + Block8x16, + Block16x8, + Block16x16, + Block16x32, + Block32x16, + Block32x32, + Block32x64, + Block64x32, + Block64x64, + Block64x128, + Block128x64, + Block128x128, + Block4x16, + Block16x4, + Block8x32, + Block32x8, + Block16x64, + Block64x16, + BlockSizeSAll, + BlockSizeS = Block4x16, + BlockInvalid = 255, + BlockLargest = BlockSizeS - 1, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1ColorFormat.cs b/src/ImageSharp/Formats/Heif/Av1/Av1ColorFormat.cs new file mode 100644 index 0000000000..07be6a0442 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1ColorFormat.cs @@ -0,0 +1,12 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal enum Av1ColorFormat +{ + Yuv400, + Yuv420, + Yuv422, + Yuv444, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1MainParseContext.cs b/src/ImageSharp/Formats/Heif/Av1/Av1MainParseContext.cs new file mode 100644 index 0000000000..4b13a86cd7 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1MainParseContext.cs @@ -0,0 +1,8 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal class Av1MainParseContext +{ +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs new file mode 100644 index 0000000000..c98652e048 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs @@ -0,0 +1,108 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal static class Av1Math +{ + public static uint MostSignificantBit(uint value) => value >> 31; + + public static uint Log2(uint n) + { + uint result = 0U; + while ((n >>= 1) > 0) + { + result++; + } + + return result; + } + + public static int Log2(int n) + { + int result = 0; + while ((n >>= 1) > 0) + { + result++; + } + + return result; + } + + public static uint FloorLog2(uint value) + { + uint s = 0; + while (value != 0U) + { + value >>= 1; + s++; + } + + return s - 1; + } + + public static uint CeilLog2(uint value) + { + if (value < 2) + { + return 0; + } + + uint i = 1; + uint p = 2; + while (p < value) + { + i++; + p <<= 1; + } + + return i; + } + + public static uint Clip1(uint value, int bitDepth) => + Clip3(0, (1U << bitDepth) - 1, value); + + public static uint Clip3(uint x, uint y, uint z) + { + if (z < x) + { + return x; + } + + if (z > y) + { + return y; + } + + return z; + } + + public static uint Round2(uint value, int n) + { + if (n == 0) + { + return value; + } + + return (uint)((value + (1 << (n - 1))) >> n); + } + + public static int Round2(int value, int n) + { + if (value < 0) + { + value = -value; + } + + return (int)Round2((uint)value, n); + } + + internal static int AlignPowerOf2(int value, int n) + { + int mask = (1 << n) - 1; + return (value + mask) & ~mask; + } + + internal static int Clamp(int value, int low, int high) + => value < low ? low : (value > high ? high : value); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Plane.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Plane.cs new file mode 100644 index 0000000000..9d73cda42a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Plane.cs @@ -0,0 +1,11 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal enum Av1Plane : int +{ + Y = 0, + U = 1, + V = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs new file mode 100644 index 0000000000..bd71ea4334 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuChromoSamplePosition.cs @@ -0,0 +1,22 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuChromoSamplePosition : byte +{ + /// + /// Unknown. + /// + Unknown = 0, + + /// + /// Horizontally co-located with luma(0, 0) sample, between two vertical samples. + /// + Vertical = 1, + + /// + /// Co-located with luma(0, 0) sample + /// + Colocated = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorConfig.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorConfig.cs new file mode 100644 index 0000000000..feafbaccd0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorConfig.cs @@ -0,0 +1,52 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuColorConfig +{ + internal bool IsColorDescriptionPresent { get; set; } + + internal int ChannelCount { get; set; } + + internal bool Monochrome { get; set; } + + internal ObuColorPrimaries ColorPrimaries { get; set; } + + internal ObuTransferCharacteristics TransferCharacteristics { get; set; } + + internal ObuMatrixCoefficients MatrixCoefficients { get; set; } + + internal bool ColorRange { get; set; } + + internal bool SubSamplingX { get; set; } + + internal bool SubSamplingY { get; set; } + + internal bool HasSeparateUvDelta { get; set; } + + internal ObuChromoSamplePosition ChromaSamplePosition { get; set; } + + internal int BitDepth { get; set; } + + internal bool HasSeparateUvDeltaQ { get; set; } + + public Av1ColorFormat GetColorFormat() + { + Av1ColorFormat format = Av1ColorFormat.Yuv400; + if (this.SubSamplingX && this.SubSamplingY) + { + format = Av1ColorFormat.Yuv420; + } + else if (this.SubSamplingX & !this.SubSamplingY) + { + format = Av1ColorFormat.Yuv422; + } + else if (!this.SubSamplingX && !this.SubSamplingY) + { + format = Av1ColorFormat.Yuv444; + } + + return format; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorPrimaries.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorPrimaries.cs new file mode 100644 index 0000000000..3136bba383 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuColorPrimaries.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuColorPrimaries +{ + None = 0, + Bt709 = 1, + Unspecified = 2, + Bt470M = 4, + Bt470BG = 5, + Bt601 = 6, + Smpte240 = 7, + GenericFilm = 8, + Bt2020 = 9, + Xyz = 10, + Smpte431 = 11, + Smpte432 = 12, + Ebu3213 = 22, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs new file mode 100644 index 0000000000..678b49d8b8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs @@ -0,0 +1,82 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal static class ObuConstants +{ + public const ObuSequenceProfile MaxSequenceProfile = ObuSequenceProfile.Professional; + + public const int LevelBits = -1; + + /// + /// Number of fractional bits for computing position in upscaling. + /// + public const int SuperResolutionScaleBits = 14; + + public const int ScaleNumerator = -1; + + /// + /// Number of reference frames that can be used for inter prediction. + /// + public const int ReferencesPerFrame = 7; + + /// + /// Maximum area of a tile in units of luma samples. + /// + public const int MaxTileArea = 4096 * 2304; + + /// + /// Maximum width of a tile in units of luma samples. + /// + public const int MaxTileWidth = 4096; + + /// + /// Maximum number of tile columns. + /// + public const int MaxTileColumnCount = 64; + + /// + /// Maximum number of tile rows. + /// + public const int MaxTileRowCount = 64; + + /// + /// Number of frames that can be stored for future reference. + /// + public const int ReferenceFrameCount = 8; + + /// + /// Value of 'PrimaryReferenceFrame' indicating that there is no primary reference frame. + /// + public const uint PrimaryReferenceFrameNone = 7; + + public const int PimaryReferenceBits = -1; + + /// + /// Number of segments allowed in segmentation map. + /// + public const int MaxSegmentCount = 8; + + /// + /// Smallest denominator for upscaling ratio. + /// + public const int SuperResolutionScaleDenominatorMinimum = 9; + + /// + /// Base 2 logarithm of maximum size of a superblock in luma samples. + /// + public const int MaxSuperBlockSizeLog2 = 7; + + /// + /// Base 2 logarithm of smallest size of a mode info block. + /// + public const int ModeInfoSizeLog2 = 2; + + public const int MaxQ = 255; + + /// + /// Number of segmentation features. + /// + public const int SegmentationLevelMax = 8; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs new file mode 100644 index 0000000000..57e54a6450 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs @@ -0,0 +1,13 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuConstraintDirectionalEnhancementFilterParameters +{ + public bool BitCount { get; internal set; } + + public int[] YStrength { get; internal set; } + + public int[] UVStrength { get; internal set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs new file mode 100644 index 0000000000..6b74c979dd --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs @@ -0,0 +1,8 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuFilmGrainParameters +{ +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs new file mode 100644 index 0000000000..03870ca56c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -0,0 +1,81 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuFrameHeader +{ + public bool ForceIntegerMotionVector { get; set; } + + public bool AllowIntraBlockCopy { get; set; } + + public bool UseReferenceFrameMotionVectors { get; set; } + + public bool AllowHighPrecisionMotionVector { get; set; } + + public ObuTileInfo TilesInfo { get; internal set; } = new ObuTileInfo(); + + public bool CodedLossless { get; internal set; } + + public bool[] LosslessArray { get; internal set; } = new bool[ObuConstants.MaxSegmentCount]; + + public ObuQuantizationParameters QuantizationParameters { get; set; } = new ObuQuantizationParameters(); + + public ObuSegmentationParameters SegmentationParameters { get; set; } = new ObuSegmentationParameters(); + + public bool AllLossless { get; internal set; } + + public bool AllowWarpedMotion { get; internal set; } + + public ObuReferenceMode ReferenceMode { get; internal set; } + + public ObuFilmGrainParameters FilmGrainParameters { get; internal set; } = new ObuFilmGrainParameters(); + + public bool ReducedTxSet { get; internal set; } + + public ObuLoopFilterParameters LoopFilterParameters { get; internal set; } = new ObuLoopFilterParameters(); + + public ObuLoopRestorationParameters[] LoopRestorationParameters { get; internal set; } = new ObuLoopRestorationParameters[3]; + + public ObuConstraintDirectionalEnhancementFilterParameters ConstraintDirectionalEnhancementFilterParameters { get; internal set; } = new ObuConstraintDirectionalEnhancementFilterParameters(); + + public int ModeInfoStride { get; internal set; } + + public bool DisableFrameEndUpdateCdf { get; internal set; } + + internal ObuFrameSize FrameSize { get; set; } = new ObuFrameSize(); + + internal int ModeInfoColumnCount { get; set; } + + internal int ModeInfoRowCount { get; set; } + + internal bool ShowExistingFrame { get; set; } + + internal ObuFrameType FrameType { get; set; } + + internal bool[] ReferenceValid { get; set; } = new bool[ObuConstants.ReferenceFrameCount]; + + internal bool[] ReferenceOrderHint { get; set; } = new bool[ObuConstants.ReferenceFrameCount]; + + internal bool ShowFrame { get; set; } + + internal bool ShowableFrame { get; set; } + + internal bool ErrorResilientMode { get; set; } + + internal bool AllowScreenContentTools { get; set; } + + internal bool DisableCdfUpdate { get; set; } + + internal bool ForeceIntegerMotionVector { get; set; } + + internal uint CurrentFrameId { get; set; } + + internal uint[] ReferenceFrameIndex { get; set; } = new uint[ObuConstants.ReferenceFrameCount]; + + internal uint OrderHint { get; set; } + + internal uint PrimaryReferenceFrame { get; set; } = ObuConstants.PrimaryReferenceFrameNone; + + internal uint RefreshFrameFlags { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs new file mode 100644 index 0000000000..7075b50b82 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuFrameSize +{ + internal int FrameWidth { get; set; } + + internal int FrameHeight { get; set; } + + internal int SuperResolutionDenominator { get; set; } + + internal int SuperResolutionUpscaledWidth { get; set; } + + internal int RenderWidth { get; set; } + + internal int RenderHeight { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameType.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameType.cs new file mode 100644 index 0000000000..eb2414edc7 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameType.cs @@ -0,0 +1,12 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuFrameType +{ + KeyFrame = 0, + InterFrame = 1, + IntraOnlyFrame = 2, + SwitchFrame = 3, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuHeader.cs new file mode 100644 index 0000000000..f55d3eb501 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuHeader.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuHeader +{ + public int Size { get; set; } + + public ObuType Type { get; set; } + + public bool HasSize { get; set; } + + public bool HasExtension { get; set; } + + public int TemporalId { get; set; } + + public int SpatialId { get; set; } + + public int PayloadSize { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs new file mode 100644 index 0000000000..961f3b2bc6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs @@ -0,0 +1,9 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuLoopFilterParameters +{ + public bool[] FilterLevel { get; internal set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs new file mode 100644 index 0000000000..1c04072e3e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs @@ -0,0 +1,9 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuLoopRestorationParameters +{ + public ObuRestorationType FrameRestorationType { get; internal set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMatrixCoefficients.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMatrixCoefficients.cs new file mode 100644 index 0000000000..e9658997fb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMatrixCoefficients.cs @@ -0,0 +1,22 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuMatrixCoefficients +{ + Identity = 0, + Bt407 = 1, + Unspecified = 2, + Fcc = 4, + Bt470BG = 5, + Bt601 = 6, + Smpte240 = 7, + SmpteYCgCo = 8, + Bt2020NonConstantLuminance = 9, + Bt2020ConstantLuminance = 10, + Smpte2085 = 11, + ChromaticityDerivedNonConstantLuminance = 12, + ChromaticityDerivedConstandLuminance = 13, + Bt2100ICtCp = 14, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMetadataType.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMetadataType.cs new file mode 100644 index 0000000000..d6788f1745 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuMetadataType.cs @@ -0,0 +1,13 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuMetadataType +{ + ItutT35, + HdrCll, + HdrMdcv, + Scalability, + Timecode +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs new file mode 100644 index 0000000000..a1401b1aa8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs @@ -0,0 +1,17 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuOperatingPoint +{ + internal int OperatorIndex { get; set; } + + internal int SequenceLevelIndex { get; set; } + + internal int SequenceTier { get; set; } + + internal bool IsDecoderModelPresent { get; set; } + + internal bool IsInitialDisplayDelayPresent { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs new file mode 100644 index 0000000000..4cdcb26550 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs @@ -0,0 +1,13 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuOrderHintInfo +{ + internal bool EnableJointCompound { get; set; } + + internal bool EnableReferenceFrameMotionVectors { get; set; } + + internal int OrderHintBits { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuQuantizationParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuQuantizationParameters.cs new file mode 100644 index 0000000000..53aa1c88bb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuQuantizationParameters.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuQuantizationParameters +{ + public int BaseQIndex { get; set; } + + public int[] QIndex { get; set; } = new int[ObuConstants.MaxSegmentCount]; + + public bool IsUsingQMatrix { get; internal set; } + + public int[] DeltaQDc { get; internal set; } = new int[3]; + + public int[] DeltaQAc { get; internal set; } = new int[3]; + + public int[] QMatrix { get; internal set; } = new int[3]; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs new file mode 100644 index 0000000000..c1676febcb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -0,0 +1,888 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuReader +{ + private static readonly int[] WienerTapsMid = new[] { 3, -7, 15 }; + private static readonly int[] SgrprojXqdMid = new[] { -32, 31 }; + + private static ObuHeader ReadObuHeader(Av1BitStreamReader reader) + { + ObuHeader header = new(); + if (!reader.ReadBoolean()) + { + throw new ImageFormatException("Forbidden bit in header should be unset."); + } + + header.Type = (ObuType)reader.ReadLiteral(4); + header.HasExtension = reader.ReadBoolean(); + header.HasSize = reader.ReadBoolean(); + if (!reader.ReadBoolean()) + { + throw new ImageFormatException("Reserved bit in header should be unset."); + } + + if (header.HasExtension) + { + header.Size++; + header.TemporalId = (int)reader.ReadLiteral(3); + header.SpatialId = (int)reader.ReadLiteral(3); + if (reader.ReadLiteral(3) != 0u) + { + throw new ImageFormatException("Reserved bits in header extension should be unset."); + } + } + else + { + header.SpatialId = 0; + header.TemporalId = 0; + } + + return header; + } + + private static void ReadObuSize(Av1BitStreamReader reader, out int obuSize) + { + ulong rawSize = reader.ReadLittleEndianBytes128(out _); + if (rawSize > uint.MaxValue) + { + throw new ImageFormatException("OBU block too large."); + } + + obuSize = (int)rawSize; + } + + /// + /// Read OBU header and size. + /// + private static ObuHeader ReadObuHeaderSize(Av1BitStreamReader reader) + { + ObuHeader header = ReadObuHeader(reader); + if (header.HasSize) + { + ReadObuSize(reader, out int payloadSize); + header.PayloadSize = payloadSize; + } + + return header; + } + + /// + /// Check that the trailing bits start with a 1 and end with 0s. + /// + /// Consumes a byte, if already byte aligned before the check. + private static void ReadTrailingBits(Av1BitStreamReader reader) + { + int bitsBeforeAlignment = 8 - (reader.BitPosition & 0x7); + uint trailing = reader.ReadLiteral(bitsBeforeAlignment); + if (trailing != (1 << (bitsBeforeAlignment - 1))) + { + throw new ImageFormatException("Trailing bits not properly formatted."); + } + } + + private static void AlignToByteBoundary(Av1BitStreamReader reader) + { + while ((reader.BitPosition & 0x7) > 0) + { + if (reader.ReadBoolean()) + { + throw new ImageFormatException("Incorrect byte alignment padding bits."); + } + } + } + + private static void ComputeImageSize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + { + frameInfo.ModeInfoColumnCount = 2 * ((frameInfo.FrameSize.FrameWidth + 7) >> 3); + frameInfo.ModeInfoRowCount = 2 * ((frameInfo.FrameSize.FrameHeight + 7) >> 3); + frameInfo.ModeInfoStride = Av1Math.AlignPowerOf2(sequenceHeader.MaxFrameWidth, ObuConstants.MaxSuperBlockSizeLog2) >> ObuConstants.ModeInfoSizeLog2; + } + + private static bool IsValidObuType(ObuType type) => type switch + { + ObuType.SequenceHeader or ObuType.TemporalDelimiter or ObuType.FrameHeader or + ObuType.TileGroup or ObuType.Metadata or ObuType.Frame or ObuType.RedundantFrameHeader or + ObuType.TileList or ObuType.Padding => true, + _ => false, + }; + + private static ObuSequenceHeader ReadSequenceHeader(Av1BitStreamReader reader) + { + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.SequenceProfile = (ObuSequenceProfile)reader.ReadLiteral(3); + if (sequenceHeader.SequenceProfile > ObuConstants.MaxSequenceProfile) + { + throw new ImageFormatException("Unknown sequence profile."); + } + + sequenceHeader.IsStillPicture = reader.ReadBoolean(); + sequenceHeader.IsReducedStillPictureHeader = reader.ReadBoolean(); + if (!sequenceHeader.IsStillPicture || !sequenceHeader.IsReducedStillPictureHeader) + { + throw new ImageFormatException("Not a picture header, is this a movie file ??"); + } + + sequenceHeader.TimingInfo = null; + sequenceHeader.DecoderModelInfoPresentFlag = false; + sequenceHeader.InitialDisplayDelayPresentFlag = false; + sequenceHeader.OperatingPoint[0].OperatorIndex = 0; + sequenceHeader.OperatingPoint[0].SequenceLevelIndex = (int)reader.ReadLiteral(ObuConstants.LevelBits); + if (!IsValidSequenceLevel(sequenceHeader.OperatingPoint[0].SequenceLevelIndex)) + { + throw new ImageFormatException("Unknown sequnce level."); + } + + sequenceHeader.OperatingPoint[0].SequenceTier = 0; + sequenceHeader.OperatingPoint[0].IsDecoderModelPresent = false; + sequenceHeader.OperatingPoint[0].IsInitialDisplayDelayPresent = false; + + // Video related flags removed + + // SVT-TODO: int operatingPoint = this.ChooseOperatingPoint(); + // sequenceHeader.OperatingPointIndex = (int)operatingPointIndices[operatingPoint]; + sequenceHeader.FrameWidthBits = (int)reader.ReadLiteral(4) + 1; + sequenceHeader.FrameHeightBits = (int)reader.ReadLiteral(4) + 1; + sequenceHeader.MaxFrameWidth = (int)reader.ReadLiteral(sequenceHeader.FrameWidthBits) + 1; + sequenceHeader.MaxFrameHeight = (int)reader.ReadLiteral(sequenceHeader.FrameHeightBits) + 1; + sequenceHeader.IsFrameIdNumbersPresent = false; + + // Video related flags removed + sequenceHeader.Use128x128SuperBlock = reader.ReadBoolean(); + sequenceHeader.SuperBlockSize = sequenceHeader.Use128x128SuperBlock ? Av1BlockSize.Block128x128 : Av1BlockSize.Block64x64; + sequenceHeader.ModeInfoSize = sequenceHeader.Use128x128SuperBlock ? 32 : 16; + sequenceHeader.SuperBlockSizeLog2 = sequenceHeader.Use128x128SuperBlock ? 7 : 6; + sequenceHeader.FilterIntraLevel = (int)reader.ReadLiteral(1); + sequenceHeader.EnableIntraEdgeFilter = reader.ReadBoolean(); + sequenceHeader.EnableInterIntraCompound = false; + sequenceHeader.EnableMaskedCompound = false; + sequenceHeader.EnableWarpedMotion = false; + sequenceHeader.EnableDualFilter = false; + sequenceHeader.OrderHintInfo.EnableJointCompound = false; + sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors = false; + sequenceHeader.SequenceForceScreenContentTools = 2; + sequenceHeader.SequenceForceIntegerMotionVector = 2; + sequenceHeader.OrderHintInfo.OrderHintBits = 0; + + // Video related flags removed + sequenceHeader.EnableSuperResolution = reader.ReadBoolean(); + sequenceHeader.CdefLevel = (int)reader.ReadLiteral(1); + sequenceHeader.EnableRestoration = reader.ReadBoolean(); + sequenceHeader.ColorConfig = ReadColorConfig(reader, sequenceHeader); + sequenceHeader.AreFilmGrainingParametersPresent = reader.ReadBoolean(); + ReadTrailingBits(reader); + return sequenceHeader; + } + + private static ObuColorConfig ReadColorConfig(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader) + { + ObuColorConfig colorConfig = new(); + ReadBitDepth(reader, colorConfig, sequenceHeader); + colorConfig.Monochrome = false; + if (sequenceHeader.SequenceProfile == ObuSequenceProfile.High) + { + colorConfig.Monochrome = reader.ReadBoolean(); + } + + colorConfig.ChannelCount = colorConfig.Monochrome ? 1 : 3; + colorConfig.IsColorDescriptionPresent = reader.ReadBoolean(); + colorConfig.ColorPrimaries = ObuColorPrimaries.Unspecified; + colorConfig.TransferCharacteristics = ObuTransferCharacteristics.Unspecified; + colorConfig.MatrixCoefficients = ObuMatrixCoefficients.Unspecified; + if (colorConfig.IsColorDescriptionPresent) + { + colorConfig.ColorPrimaries = (ObuColorPrimaries)reader.ReadLiteral(8); + colorConfig.TransferCharacteristics = (ObuTransferCharacteristics)reader.ReadLiteral(8); + colorConfig.MatrixCoefficients = (ObuMatrixCoefficients)reader.ReadLiteral(8); + } + + colorConfig.ColorRange = false; + colorConfig.SubSamplingX = false; + colorConfig.SubSamplingY = false; + colorConfig.ChromaSamplePosition = ObuChromoSamplePosition.Unknown; + colorConfig.HasSeparateUvDelta = false; + if (colorConfig.Monochrome) + { + colorConfig.ColorRange = reader.ReadBoolean(); + colorConfig.SubSamplingX = true; + colorConfig.SubSamplingY = true; + return colorConfig; + } + else if ( + colorConfig.ColorPrimaries == ObuColorPrimaries.Bt709 && + colorConfig.TransferCharacteristics == ObuTransferCharacteristics.Srgb && + colorConfig.MatrixCoefficients == ObuMatrixCoefficients.Identity) + { + colorConfig.ColorRange = true; + colorConfig.SubSamplingX = false; + colorConfig.SubSamplingY = false; + } + else + { + colorConfig.ColorRange = reader.ReadBoolean(); + if (sequenceHeader.SequenceProfile != ObuSequenceProfile.Main) + { + if (colorConfig.BitDepth == 12) + { + colorConfig.SubSamplingX = reader.ReadBoolean(); + if (colorConfig.SubSamplingX) + { + colorConfig.SubSamplingY = reader.ReadBoolean(); + } + } + else + { + colorConfig.SubSamplingX = true; + colorConfig.SubSamplingY = false; + } + } + + if (colorConfig.SubSamplingX && colorConfig.SubSamplingY) + { + colorConfig.ChromaSamplePosition = (ObuChromoSamplePosition)reader.ReadLiteral(2); + } + } + + colorConfig.HasSeparateUvDeltaQ = reader.ReadBoolean(); + return colorConfig; + } + + private static void ReadBitDepth(Av1BitStreamReader reader, ObuColorConfig colorConfig, ObuSequenceHeader sequenceHeader) + { + bool hasHighBitDepth = reader.ReadBoolean(); + if (sequenceHeader.SequenceProfile == ObuSequenceProfile.Professional && hasHighBitDepth) + { + colorConfig.BitDepth = reader.ReadBoolean() ? 12 : 10; + } + else if (sequenceHeader.SequenceProfile <= ObuSequenceProfile.Professional) + { + colorConfig.BitDepth = hasHighBitDepth ? 10 : 8; + } + } + + private static void ReadSuperResolutionParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + { + bool useSuperResolution = false; + if (sequenceHeader.EnableSuperResolution) + { + useSuperResolution = reader.ReadBoolean(); + } + + if (useSuperResolution) + { + frameInfo.FrameSize.SuperResolutionDenominator = (int)reader.ReadLiteral(ObuConstants.SuperResolutionScaleBits) + ObuConstants.SuperResolutionScaleDenominatorMinimum; + } + else + { + frameInfo.FrameSize.SuperResolutionDenominator = ObuConstants.ScaleNumerator; + } + + frameInfo.FrameSize.SuperResolutionUpscaledWidth = frameInfo.FrameSize.FrameWidth; + frameInfo.FrameSize.FrameWidth = + (frameInfo.FrameSize.SuperResolutionUpscaledWidth * ObuConstants.ScaleNumerator) + + (frameInfo.FrameSize.SuperResolutionDenominator / 2); + + if (frameInfo.FrameSize.SuperResolutionDenominator != ObuConstants.ScaleNumerator) + { + int manWidth = Math.Min(16, frameInfo.FrameSize.SuperResolutionUpscaledWidth); + frameInfo.FrameSize.FrameWidth = Math.Max(manWidth, frameInfo.FrameSize.FrameWidth); + } + } + + private static void ReadRenderSize(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + { + bool renderSizeAndFrameSizeDifferent = reader.ReadBoolean(); + if (renderSizeAndFrameSizeDifferent) + { + frameInfo.FrameSize.RenderWidth = (int)reader.ReadLiteral(16) + 1; + frameInfo.FrameSize.RenderHeight = (int)reader.ReadLiteral(16) + 1; + } + else + { + frameInfo.FrameSize.RenderWidth = frameInfo.FrameSize.SuperResolutionUpscaledWidth; + frameInfo.FrameSize.RenderHeight = frameInfo.FrameSize.FrameHeight; + } + } + + private static void ReadFrameSizeWithReferences(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool frameSizeOverrideFlag) + { + bool foundReference = false; + for (int i = 0; i < ObuConstants.ReferencesPerFrame; i++) + { + foundReference = reader.ReadBoolean(); + if (foundReference) + { + // Take values over from reference frame + break; + } + } + + if (!foundReference) + { + ReadFrameSize(reader, sequenceHeader, frameInfo, frameSizeOverrideFlag); + ReadRenderSize(reader, frameInfo); + } + else + { + ReadSuperResolutionParameters(reader, sequenceHeader, frameInfo); + ComputeImageSize(sequenceHeader, frameInfo); + } + } + + private static void ReadFrameSize(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool frameSizeOverrideFlag) + { + if (frameSizeOverrideFlag) + { + frameInfo.FrameSize.FrameWidth = (int)reader.ReadLiteral(sequenceHeader.FrameWidthBits) + 1; + frameInfo.FrameSize.FrameHeight = (int)reader.ReadLiteral(sequenceHeader.FrameHeightBits) + 1; + } + else + { + frameInfo.FrameSize.FrameWidth = sequenceHeader.MaxFrameWidth; + frameInfo.FrameSize.FrameHeight = sequenceHeader.MaxFrameHeight; + } + + ReadSuperResolutionParameters(reader, sequenceHeader, frameInfo); + ComputeImageSize(sequenceHeader, frameInfo); + } + + private static ObuTileInfo ReadTileInfo(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + { + ObuTileInfo tileInfo = new(); + int superBlockColumnCount; + int superBlockRowCount; + int superBlockShift; + if (sequenceHeader.Use128x128SuperBlock) + { + superBlockColumnCount = (frameInfo.ModeInfoColumnCount + 31) >> 5; + superBlockRowCount = (frameInfo.ModeInfoRowCount + 31) >> 5; + superBlockShift = 5; + } + else + { + superBlockColumnCount = (frameInfo.ModeInfoColumnCount + 15) >> 4; + superBlockRowCount = (frameInfo.ModeInfoRowCount + 15) >> 4; + superBlockShift = 4; + } + + int superBlockSize = superBlockShift + 2; + int maxTileAreaOfSuperBlock = ObuConstants.MaxTileArea >> (2 * superBlockSize); + + tileInfo.MaxTileWidthSuperBlock = ObuConstants.MaxTileWidth >> superBlockSize; + tileInfo.MaxTileHeightSuperBlock = (ObuConstants.MaxTileArea / ObuConstants.MaxTileWidth) >> superBlockSize; + tileInfo.MinLog2TileColumnCount = TileLog2(tileInfo.MaxTileWidthSuperBlock, superBlockColumnCount); + tileInfo.MaxLog2TileColumnCount = TileLog2(1, Math.Min(superBlockColumnCount, ObuConstants.MaxTileColumnCount)); + tileInfo.MaxLog2TileRowCount = TileLog2(1, Math.Min(superBlockRowCount, ObuConstants.MaxTileRowCount)); + tileInfo.MinLog2TileCount = Math.Max(tileInfo.MinLog2TileColumnCount, TileLog2(maxTileAreaOfSuperBlock, superBlockColumnCount * superBlockRowCount)); + tileInfo.HasUniformTileSpacing = reader.ReadBoolean(); + if (tileInfo.HasUniformTileSpacing) + { + tileInfo.TileColumnCountLog2 = tileInfo.MinLog2TileColumnCount; + while (tileInfo.TileColumnCountLog2 < tileInfo.MaxLog2TileColumnCount) + { + if (reader.ReadBoolean()) + { + tileInfo.TileColumnCountLog2++; + } + else + { + break; + } + } + + int tileWidthSuperBlock = (superBlockColumnCount + (1 << tileInfo.TileColumnCountLog2) - 1) >> tileInfo.TileColumnCountLog2; + if (tileWidthSuperBlock > tileInfo.MaxTileWidthSuperBlock) + { + throw new ImageFormatException("Invalid tile width specified."); + } + + int i = 0; + tileInfo.TileColumnStartModeInfo = new int[superBlockColumnCount + 1]; + for (int startSuperBlock = 0; startSuperBlock < superBlockColumnCount; startSuperBlock += tileWidthSuperBlock) + { + tileInfo.TileColumnStartModeInfo[i] = startSuperBlock << superBlockShift; + i++; + } + + tileInfo.TileColumnStartModeInfo[i] = frameInfo.ModeInfoColumnCount; + tileInfo.TileColumnCount = i; + + tileInfo.MinLog2TileRowCount = Math.Max(tileInfo.MinLog2TileCount - tileInfo.TileColumnCountLog2, 0); + tileInfo.TileRowCountLog2 = tileInfo.MinLog2TileRowCount; + while (tileInfo.TileRowCountLog2 < tileInfo.MaxLog2TileRowCount) + { + if (reader.ReadBoolean()) + { + tileInfo.TileRowCountLog2++; + } + else + { + break; + } + } + + int tileHeightSuperBlock = (superBlockRowCount + (1 << tileInfo.TileRowCountLog2) - 1) >> tileInfo.TileRowCountLog2; + if (tileHeightSuperBlock > tileInfo.MaxTileHeightSuperBlock) + { + throw new ImageFormatException("Invalid tile height specified."); + } + + i = 0; + tileInfo.TileRowStartModeInfo = new int[superBlockRowCount + 1]; + for (int startSuperBlock = 0; startSuperBlock < superBlockRowCount; startSuperBlock += tileHeightSuperBlock) + { + tileInfo.TileRowStartModeInfo[i] = startSuperBlock << superBlockShift; + i++; + } + + tileInfo.TileRowStartModeInfo[i] = frameInfo.ModeInfoRowCount; + tileInfo.TileRowCount = i; + } + else + { + uint widestTileSuperBlock = 0U; + int startSuperBlock = 0; + int i = 0; + for (; startSuperBlock < superBlockColumnCount; i++) + { + tileInfo.TileColumnStartModeInfo[i] = startSuperBlock << superBlockShift; + uint maxWidth = (uint)Math.Min(superBlockColumnCount - startSuperBlock, tileInfo.MaxTileWidthSuperBlock); + uint widthInSuperBlocks = reader.ReadNonSymmetric(maxWidth) + 1; + widestTileSuperBlock = Math.Max(widthInSuperBlocks, widestTileSuperBlock); + startSuperBlock += (int)widthInSuperBlocks; + } + + if (startSuperBlock != superBlockColumnCount) + { + throw new ImageFormatException("Super block tiles width does not add up to total width."); + } + + tileInfo.TileColumnStartModeInfo[i] = frameInfo.ModeInfoColumnCount; + tileInfo.TileColumnCount = i; + tileInfo.TileColumnCountLog2 = TileLog2(1, tileInfo.TileColumnCount); + if (tileInfo.MinLog2TileCount > 0) + { + maxTileAreaOfSuperBlock = (superBlockRowCount * superBlockColumnCount) >> (tileInfo.MinLog2TileCount + 1); + } + else + { + maxTileAreaOfSuperBlock = superBlockRowCount * superBlockColumnCount; + } + + DebugGuard.MustBeGreaterThan(widestTileSuperBlock, 0U, nameof(widestTileSuperBlock)); + tileInfo.MaxTileHeightSuperBlock = Math.Max(maxTileAreaOfSuperBlock / (int)widestTileSuperBlock, 1); + + startSuperBlock = 0; + for (i = 0; startSuperBlock < superBlockRowCount; i++) + { + tileInfo.TileRowStartModeInfo[i] = startSuperBlock << superBlockShift; + uint maxHeight = (uint)Math.Min(superBlockRowCount - startSuperBlock, tileInfo.MaxTileHeightSuperBlock); + uint heightInSuperBlocks = reader.ReadNonSymmetric(maxHeight) + 1; + startSuperBlock += (int)heightInSuperBlocks; + } + + if (startSuperBlock != superBlockRowCount) + { + throw new ImageFormatException("Super block tiles height does not add up to total height."); + } + + tileInfo.TileRowStartModeInfo[i] = frameInfo.ModeInfoRowCount; + tileInfo.TileRowCount = i; + tileInfo.TileRowCountLog2 = TileLog2(1, tileInfo.TileRowCount); + } + + if (tileInfo.TileColumnCount > ObuConstants.MaxTileColumnCount || tileInfo.TileRowCount > ObuConstants.MaxTileRowCount) + { + throw new ImageFormatException("Tile width or height too big."); + } + + if (tileInfo.TileColumnCountLog2 > 0 || tileInfo.TileRowCountLog2 > 0) + { + tileInfo.ContextUpdateTileId = reader.ReadLiteral(tileInfo.TileRowCountLog2 + tileInfo.TileColumnCountLog2); + tileInfo.TileSizeBytes = (int)reader.ReadLiteral(2) + 1; + } + else + { + tileInfo.ContextUpdateTileId = 0; + } + + if (tileInfo.ContextUpdateTileId >= (tileInfo.TileColumnCount * tileInfo.TileRowCount)) + { + throw new ImageFormatException("Context update Tile ID too large."); + } + + return tileInfo; + } + + private static void ReadUncompressedFrameHeader(Av1BitStreamReader reader, ObuHeader header, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + { + int idLength = 0; + uint previousFrameId = 0; + bool isIntraFrame = false; + bool frameSizeOverrideFlag = false; + if (sequenceHeader.IsFrameIdNumbersPresent) + { + idLength = sequenceHeader.FrameIdLength - 1 + sequenceHeader.DeltaFrameIdLength - 2 + 3; + DebugGuard.MustBeLessThanOrEqualTo(idLength, 16, nameof(idLength)); + } + + if (sequenceHeader.IsReducedStillPictureHeader) + { + frameInfo.ShowExistingFrame = false; + frameInfo.FrameType = ObuFrameType.KeyFrame; + isIntraFrame = true; + frameInfo.ShowFrame = true; + frameInfo.ShowableFrame = false; + frameInfo.ErrorResilientMode = true; + } + + if (frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame) + { + frameInfo.ReferenceValid = new bool[ObuConstants.ReferenceFrameCount]; + frameInfo.ReferenceOrderHint = new bool[ObuConstants.ReferenceFrameCount]; + for (int i = 0; i < ObuConstants.ReferenceFrameCount; i++) + { + frameInfo.ReferenceValid[i] = false; + frameInfo.ReferenceOrderHint[i] = false; + } + } + + frameInfo.DisableCdfUpdate = reader.ReadBoolean(); + frameInfo.AllowScreenContentTools = sequenceHeader.SequenceForceScreenContentTools == 1; + if (frameInfo.AllowScreenContentTools) + { + frameInfo.AllowScreenContentTools = reader.ReadBoolean(); + } + + if (frameInfo.AllowScreenContentTools) + { + if (sequenceHeader.SequenceForceIntegerMotionVector == 1) + { + frameInfo.ForeceIntegerMotionVector = reader.ReadBoolean(); + } + else + { + frameInfo.ForceIntegerMotionVector = sequenceHeader.SequenceForceIntegerMotionVector != 0; + } + } + else + { + frameInfo.ForeceIntegerMotionVector = false; + } + + if (isIntraFrame) + { + frameInfo.ForeceIntegerMotionVector = true; + } + + bool havePreviousFrameId = !(frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame); + if (havePreviousFrameId) + { + previousFrameId = frameInfo.CurrentFrameId; + } + + if (sequenceHeader.IsFrameIdNumbersPresent) + { + frameInfo.CurrentFrameId = reader.ReadLiteral(idLength); + if (havePreviousFrameId) + { + uint diffFrameId = (frameInfo.CurrentFrameId > previousFrameId) ? + frameInfo.CurrentFrameId - previousFrameId : + (uint)((1 << idLength) + (int)frameInfo.CurrentFrameId - previousFrameId); + if (frameInfo.CurrentFrameId == previousFrameId || diffFrameId >= 1 << (idLength - 1)) + { + throw new ImageFormatException("Current frame ID cannot be same as previous Frame ID"); + } + } + + int diffLength = sequenceHeader.DeltaFrameIdLength; + for (int i = 0; i < ObuConstants.ReferenceFrameCount; i++) + { + if (frameInfo.CurrentFrameId > (1U << diffLength)) + { + if ((frameInfo.ReferenceFrameIndex[i] > frameInfo.CurrentFrameId) || + frameInfo.ReferenceFrameIndex[i] > (frameInfo.CurrentFrameId - (1 - diffLength))) + { + frameInfo.ReferenceValid[i] = false; + } + } + else if (frameInfo.ReferenceFrameIndex[i] > frameInfo.CurrentFrameId && + frameInfo.ReferenceFrameIndex[i] < ((1 << idLength) + (frameInfo.CurrentFrameId - (1 << diffLength)))) + { + frameInfo.ReferenceValid[i] = false; + } + } + } + else + { + frameInfo.CurrentFrameId = 0; + } + + if (frameInfo.FrameType == ObuFrameType.SwitchFrame) + { + frameSizeOverrideFlag = true; + } + else if (sequenceHeader.IsReducedStillPictureHeader) + { + frameSizeOverrideFlag = false; + } + else + { + frameSizeOverrideFlag = reader.ReadBoolean(); + } + + frameInfo.OrderHint = reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); + if (isIntraFrame || frameInfo.ErrorResilientMode) + { + frameInfo.PrimaryReferenceFrame = ObuConstants.PrimaryReferenceFrameNone; + } + else + { + frameInfo.PrimaryReferenceFrame = reader.ReadLiteral(ObuConstants.PimaryReferenceBits); + } + + // Skipping, as no decoder info model present + frameInfo.AllowHighPrecisionMotionVector = false; + frameInfo.UseReferenceFrameMotionVectors = false; + frameInfo.AllowIntraBlockCopy = false; + if (frameInfo.FrameType == ObuFrameType.SwitchFrame || (frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame)) + { + frameInfo.RefreshFrameFlags = 0xffU; + } + else + { + frameInfo.RefreshFrameFlags = reader.ReadLiteral(8); + } + + if (frameInfo.FrameType == ObuFrameType.IntraOnlyFrame) + { + DebugGuard.IsTrue(frameInfo.RefreshFrameFlags != 0xFFU, nameof(frameInfo.RefreshFrameFlags)); + } + + if (!isIntraFrame || (frameInfo.RefreshFrameFlags != 0xFFU)) + { + if (frameInfo.ErrorResilientMode && sequenceHeader.OrderHintInfo != null) + { + for (int i = 0; i < ObuConstants.ReferenceFrameCount; i++) + { + int referenceOrderHint = (int)reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); + if (referenceOrderHint != (frameInfo.ReferenceOrderHint[i] ? 1U : 0U)) + { + frameInfo.ReferenceValid[i] = false; + } + } + } + } + + if (isIntraFrame) + { + ReadFrameSize(reader, sequenceHeader, frameInfo, frameSizeOverrideFlag); + ReadRenderSize(reader, frameInfo); + if (frameInfo.AllowScreenContentTools && frameInfo.FrameSize.RenderWidth != 0) + { + if (frameInfo.FrameSize.FrameWidth == frameInfo.FrameSize.SuperResolutionUpscaledWidth) + { + frameInfo.AllowIntraBlockCopy = reader.ReadBoolean(); + } + } + } + else + { + // Single image is always Intra. + } + + SetupFrameBufferReferences(sequenceHeader, frameInfo); + CheckAddTemporalMotionVectorBuffer(sequenceHeader, frameInfo); + SetupFrameSignBias(sequenceHeader, frameInfo); + + if (sequenceHeader.IsReducedStillPictureHeader || frameInfo.DisableCdfUpdate) + { + frameInfo.DisableFrameEndUpdateCdf = true; + } + + if (frameInfo.PrimaryReferenceFrame == ObuConstants.PrimaryReferenceFrameNone) + { + SetupPastIndependence(); + } + + GenerateNextReferenceFrameMap(sequenceHeader, frameInfo); + + frameInfo.TilesInfo = ReadTileInfo(reader, sequenceHeader, frameInfo); + frameInfo.QuantizationParameters = ReadQuantizationParameters(reader, sequenceHeader.ColorConfig, planesCount); + ReadSegmentationParameters(reader, sequenceHeader, frameInfo); + ReadDeltaQParameters(reader, frameInfo); + ReadDeltaLoopFilterParameters(reader, frameInfo); + SetupSegmentationDequantization(); + + Av1MainParseContext mainParseContext = new(); + if (frameInfo.PrimaryReferenceFrame == ObuConstants.PrimaryReferenceFrameNone) + { + ResetParseContext(mainParseContext, frameInfo.QuantizationParameters.BaseQIndex); + } + + int tilesCount = frameInfo.TilesInfo.TileColumnCount * frameInfo.TilesInfo.TileRowCount; + frameInfo.CodedLossless = true; + for (int segmentId = 0; segmentId < ObuConstants.MaxSegmentCount; segmentId++) + { + int qIndex = GetQIndex(frameInfo.SegmentationParameters, segmentId, frameInfo.QuantizationParameters.BaseQIndex); + frameInfo.QuantizationParameters.QIndex[segmentId] = qIndex; + frameInfo.LosslessArray[segmentId] = qIndex == 0 && + frameInfo.QuantizationParameters.DeltaQDc[(int)Av1Plane.Y] == 0 && + frameInfo.QuantizationParameters.DeltaQAc[(int)Av1Plane.U] == 0 && + frameInfo.QuantizationParameters.DeltaQDc[(int)Av1Plane.U] == 0 && + frameInfo.QuantizationParameters.DeltaQAc[(int)Av1Plane.V] == 0 && + frameInfo.QuantizationParameters.DeltaQDc[(int)Av1Plane.V] == 0; + if (!frameInfo.LosslessArray[segmentId]) + { + frameInfo.CodedLossless = false; + } + + if (frameInfo.QuantizationParameters.IsUsingQMatrix) + { + if (frameInfo.LosslessArray[segmentId]) + { + frameInfo.SegmentationParameters.QMLevel[0, segmentId] = 15; + frameInfo.SegmentationParameters.QMLevel[1, segmentId] = 15; + frameInfo.SegmentationParameters.QMLevel[2, segmentId] = 15; + } + else + { + frameInfo.SegmentationParameters.QMLevel[0, segmentId] = frameInfo.QuantizationParameters.QMatrix[(int)Av1Plane.Y]; + frameInfo.SegmentationParameters.QMLevel[1, segmentId] = frameInfo.QuantizationParameters.QMatrix[(int)Av1Plane.U]; + frameInfo.SegmentationParameters.QMLevel[2, segmentId] = frameInfo.QuantizationParameters.QMatrix[(int)Av1Plane.V]; + } + } + } + + frameInfo.AllLossless = frameInfo.CodedLossless && frameInfo.FrameSize.FrameWidth == frameInfo.FrameSize.SuperResolutionUpscaledWidth; + ReadLoopFilterParameters(reader, sequenceHeader, frameInfo, planesCount); + ReadConstraintDirectionalEnhancementFilterParameters(reader, sequenceHeader, frameInfo, planesCount); + ReadLoopRestorationParameter(reader, sequenceHeader, frameInfo, planesCount); + ReadTransformMode(reader, frameInfo); + + frameInfo.ReferenceMode = ReadFrameReferenceMode(reader, isIntraFrame); + ReadSkipModeParameters(reader, frameInfo, isIntraFrame, sequenceHeader, frameInfo.ReferenceMode); + if (isIntraFrame || frameInfo.ErrorResilientMode || !sequenceHeader.EnableWarpedMotion) + { + frameInfo.AllowWarpedMotion = false; + } + + frameInfo.ReducedTxSet = reader.ReadBoolean(); + ReadGlobalMotionParameters(reader, sequenceHeader, frameInfo, isIntraFrame); + frameInfo.FilmGrainParameters = ReadFilmGrainParameters(reader, sequenceHeader, frameInfo); + } + + private static bool IsSegmentationFeatureActive(ObuSegmentationParameters segmentationParameters, int segmentId, ObuSegmentationLevelFeature feature) + => segmentationParameters.SegmentationEnabled && segmentationParameters.FeatureEnabled[segmentId, (int)feature]; + + private static int GetQIndex(ObuSegmentationParameters segmentationParameters, int segmentId, int baseQIndex) + { + if (IsSegmentationFeatureActive(segmentationParameters, segmentId, ObuSegmentationLevelFeature.AlternativeQuantizer)) + { + int data = segmentationParameters.FeatureData[segmentId, (int)ObuSegmentationLevelFeature.AlternativeQuantizer]; + int qIndex = baseQIndex + data; + return Av1Math.Clamp(qIndex, 0, ObuConstants.MaxQ); + } + else + { + return baseQIndex; + } + } + + private static void ReadFrameHeader(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuHeader header, bool trailingBit) + { + int planeCount = sequenceHeader.ColorConfig.Monochrome ? 1 : 3; + int startBitPosition = reader.BitPosition; + ReadUncompressedFrameHeader(reader, header, sequenceHeader, frameInfo, planeCount); + if (trailingBit) + { + ReadTrailingBits(reader); + } + + AlignToByteBoundary(reader); + + int endPosition = reader.BitPosition; + int headerBytes = (endPosition - startBitPosition) / 8; + header.PayloadSize -= headerBytes; + } + + private static void ReadTileGroup(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileInfo tileInfo, ObuHeader header, out bool isLastTileGroup) + { + int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; + int startBitPosition = reader.BitPosition; + bool tileStartAndEndPresentFlag = false; + if (tileCount > 1) + { + tileStartAndEndPresentFlag = reader.ReadBoolean(); + } + + if (header.Type == ObuType.FrameHeader) + { + DebugGuard.IsFalse(tileStartAndEndPresentFlag, nameof(tileStartAndEndPresentFlag), "Frame header should not set 'tileStartAndEndPresentFlag'."); + } + + int tileGroupStart = 0; + int tileGroupEnd = tileCount - 1; + if (tileCount != 1 && tileStartAndEndPresentFlag) + { + int tileBits = Av1Math.Log2(tileInfo.TileColumnCount) + Av1Math.Log2(tileInfo.TileRowCount); + tileGroupStart = (int)reader.ReadLiteral(tileBits); + tileGroupEnd = (int)reader.ReadLiteral(tileBits); + } + + isLastTileGroup = (tileGroupEnd + 1) == tileCount; + AlignToByteBoundary(reader); + int endBitPosition = reader.BitPosition; + int headerBytes = (endBitPosition - startBitPosition) / 8; + header.PayloadSize -= headerBytes; + + bool noIbc = !frameInfo.AllowIntraBlockCopy; + bool doLoopFilter = noIbc && (frameInfo.LoopFilterParameters.FilterLevel[0] || frameInfo.LoopFilterParameters.FilterLevel[1]); + bool doCdef = noIbc && (!frameInfo.CodedLossless && + (frameInfo.ConstraintDirectionalEnhancementFilterParameters.BitCount || + frameInfo.ConstraintDirectionalEnhancementFilterParameters.YStrength[0] != 0 || + frameInfo.ConstraintDirectionalEnhancementFilterParameters.UVStrength[0] != 0)); + bool doLoopRestoration = noIbc && + (frameInfo.LoopRestorationParameters[(int)Av1Plane.Y].FrameRestorationType != ObuRestorationType.RestoreNone || + frameInfo.LoopRestorationParameters[(int)Av1Plane.U].FrameRestorationType != ObuRestorationType.RestoreNone || + frameInfo.LoopRestorationParameters[(int)Av1Plane.V].FrameRestorationType != ObuRestorationType.RestoreNone); + + for (int tileNum = tileGroupStart; tileNum <= tileGroupEnd; tileNum++) + { + int tileRow = tileNum / tileInfo.TileColumnCount; + int tileColumn = tileNum % tileInfo.TileColumnCount; + bool isLastTile = tileNum == tileGroupEnd; + int tileSize = header.PayloadSize; + if (!isLastTile) + { + tileSize = (int)reader.ReadLittleEndian(tileInfo.TileSizeBytes) + 1; + header.PayloadSize -= tileSize + tileInfo.TileSizeBytes; + } + + // TODO: Pass more info to the decoder. + DecodeTile(sequenceHeader, frameInfo, tileInfo, tileNum); + } + + if (tileGroupEnd != tileCount - 1) + { + return; + } + + FinishDecodeTiles(sequenceHeader, frameInfo, doCdef, doLoopRestoration); + } + + private static bool IsValidSequenceLevel(int sequenceLevelIndex) + => sequenceLevelIndex is < 24 or 31; + + private static int TileLog2(int blockSize, int target) + { + int k; + for (k = 0; (blockSize << k) < target; k++) + { + } + + return k; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReferenceMode.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReferenceMode.cs new file mode 100644 index 0000000000..a76bc7601e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReferenceMode.cs @@ -0,0 +1,10 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuReferenceMode +{ + ReferenceModeSelect, + SingleReference, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuRestorationType.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuRestorationType.cs new file mode 100644 index 0000000000..02ab7cd1e4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuRestorationType.cs @@ -0,0 +1,9 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuRestorationType +{ + RestoreNone +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationLevelFeature.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationLevelFeature.cs new file mode 100644 index 0000000000..93af8e0a1c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationLevelFeature.cs @@ -0,0 +1,16 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuSegmentationLevelFeature +{ + AlternativeQuantizer, + AlternativeLoopFilterYVertical, + AlternativeLoopFilterYHorizontal, + AlternativeLoopFilterU, + AlternativeLoopFilterV, + ReferenceFrame, + Skip, + GlobalMotionVector, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs new file mode 100644 index 0000000000..ed92abf4c4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs @@ -0,0 +1,15 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuSegmentationParameters +{ + public int[,] QMLevel { get; internal set; } = new int[3, ObuConstants.MaxSegmentCount]; + + public bool[,] FeatureEnabled { get; internal set; } = new bool[ObuConstants.MaxSegmentCount, ObuConstants.SegmentationLevelMax]; + + public bool SegmentationEnabled { get; internal set; } + + public int[,] FeatureData { get; internal set; } = new int[ObuConstants.MaxSegmentCount, ObuConstants.SegmentationLevelMax]; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs new file mode 100644 index 0000000000..652734c01b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs @@ -0,0 +1,71 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuSequenceHeader +{ + internal bool IsStillPicture { get; set; } + + internal bool IsReducedStillPictureHeader { get; set; } + + internal ObuSequenceProfile SequenceProfile { get; set; } + + internal ObuOperatingPoint[] OperatingPoint { get; set; } = new ObuOperatingPoint[1]; + + internal bool InitialDisplayDelayPresentFlag { get; set; } + + internal bool DecoderModelInfoPresentFlag { get; set; } + + internal object? TimingInfo { get; set; } + + internal bool IsFrameIdNumbersPresent { get; set; } + + internal int FrameWidthBits { get; set; } + + internal int FrameHeightBits { get; set; } + + internal int MaxFrameWidth { get; set; } + + internal int MaxFrameHeight { get; set; } + + internal bool Use128x128SuperBlock { get; set; } + + internal Av1BlockSize SuperBlockSize { get; set; } + + internal int ModeInfoSize { get; set; } + + internal int SuperBlockSizeLog2 { get; set; } + + internal int FilterIntraLevel { get; set; } + + internal bool EnableIntraEdgeFilter { get; set; } + + internal ObuOrderHintInfo OrderHintInfo { get; set; } = new ObuOrderHintInfo(); + + internal bool EnableInterIntraCompound { get; set; } + + internal bool EnableMaskedCompound { get; set; } + + internal bool EnableWarpedMotion { get; set; } + + internal bool EnableDualFilter { get; set; } + + internal int SequenceForceIntegerMotionVector { get; set; } + + internal int SequenceForceScreenContentTools { get; set; } + + internal bool EnableSuperResolution { get; set; } + + internal int CdefLevel { get; set; } + + internal bool EnableRestoration { get; set; } + + internal ObuColorConfig ColorConfig { get; set; } = new ObuColorConfig(); + + internal bool AreFilmGrainingParametersPresent { get; set; } + + internal int FrameIdLength { get; set; } + + internal int DeltaFrameIdLength { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceProfile.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceProfile.cs new file mode 100644 index 0000000000..f1ac85edb3 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceProfile.cs @@ -0,0 +1,11 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuSequenceProfile : uint +{ + Main = 0, + High = 1, + Professional = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs new file mode 100644 index 0000000000..24c8f963ea --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs @@ -0,0 +1,39 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuTileInfo +{ + internal int MaxTileWidthSuperBlock { get; set; } + + internal int MaxTileHeightSuperBlock { get; set; } + + internal int MinLog2TileColumnCount { get; set; } + + internal int MaxLog2TileColumnCount { get; set; } + + internal int MaxLog2TileRowCount { get; set; } + + internal int MinLog2TileCount { get; set; } + + public bool HasUniformTileSpacing { get; set; } + + internal int TileColumnCountLog2 { get; set; } + + internal int TileColumnCount { get; set; } + + internal int[] TileColumnStartModeInfo { get; set; } + + internal int MinLog2TileRowCount { get; set; } + + internal int TileRowCountLog2 { get; set; } + + internal int[] TileRowStartModeInfo { get; set; } + + internal int TileRowCount { get; set; } + + internal uint ContextUpdateTileId { get; set; } + + internal int TileSizeBytes { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTransferCharacteristics.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTransferCharacteristics.cs new file mode 100644 index 0000000000..0fa71e5922 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTransferCharacteristics.cs @@ -0,0 +1,25 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuTransferCharacteristics +{ + Bt709 = 1, + Unspecified = 2, + Bt470M = 4, + Bt470BG = 5, + Bt601 = 6, + Smpte240 = 7, + Linear = 8, + Log100 = 9, + Log100Sqrt10 = 10, + Iec61966 = 11, + Bt1361 = 12, + Srgb = 13, + Bt202010Bit = 14, + Bt202012Bit = 15, + Smpte2084 = 16, + Smpte248 = 17, + Hlg = 18, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuType.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuType.cs new file mode 100644 index 0000000000..34285ef2f0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuType.cs @@ -0,0 +1,18 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal enum ObuType +{ + None = 0, + SequenceHeader = 1, + TemporalDelimiter = 2, + FrameHeader = 3, + RedundantFrameHeader = 7, + TileGroup = 4, + Metadata = 5, + Frame = 6, + TileList = 8, + Padding = 15, +} From f93492704434c3942a8dc7772b44aaae7881b011 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Mon, 8 Jan 2024 21:36:16 +0100 Subject: [PATCH 02/12] Read parameters --- .../Formats/Heif/Av1/Av1TransformMode.cs | 11 + ...tDirectionalEnhancementFilterParameters.cs | 8 +- .../ObuDeltaLoopFilterParameters.cs | 13 + .../OpenBitstreamUnit/ObuDeltaQParameters.cs | 11 + .../ObuFilmGrainParameters.cs | 1 + .../Av1/OpenBitstreamUnit/ObuFrameHeader.cs | 34 ++- .../ObuLoopFilterParameters.cs | 2 +- .../Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs | 2 + .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 278 ++++++++++++++++-- .../ObuSkipModeParameters.cs | 11 + .../Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs | 4 +- 11 files changed, 333 insertions(+), 42 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaLoopFilterParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaQParameters.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSkipModeParameters.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs b/src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs new file mode 100644 index 0000000000..99d47f61e2 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs @@ -0,0 +1,11 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal enum Av1TransformMode : byte +{ + Only4x4 = 0, + Largest = 1, + Select = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs index 57e54a6450..044e7799e5 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs @@ -5,9 +5,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuConstraintDirectionalEnhancementFilterParameters { - public bool BitCount { get; internal set; } + public int BitCount { get; internal set; } - public int[] YStrength { get; internal set; } + public int[] YStrength { get; internal set; } = new int[4]; - public int[] UVStrength { get; internal set; } + public int[] UVStrength { get; internal set; } = new int[4]; + + public int Damping { get; internal set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaLoopFilterParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaLoopFilterParameters.cs new file mode 100644 index 0000000000..76450d968c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaLoopFilterParameters.cs @@ -0,0 +1,13 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuDeltaLoopFilterParameters +{ + public bool IsPresent { get; internal set; } + + public int Resolution { get; internal set; } + + public bool Multi { get; internal set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaQParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaQParameters.cs new file mode 100644 index 0000000000..b82bf5cf96 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuDeltaQParameters.cs @@ -0,0 +1,11 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuDeltaQParameters +{ + public bool IsPresent { get; set; } + + public int Resolution { get; set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs index 6b74c979dd..67b8f352ed 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs @@ -5,4 +5,5 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuFilmGrainParameters { + public bool ApplyGrain { get; set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs index 03870ca56c..3ccf43a36d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -13,35 +13,43 @@ internal class ObuFrameHeader public bool AllowHighPrecisionMotionVector { get; set; } - public ObuTileInfo TilesInfo { get; internal set; } = new ObuTileInfo(); + public ObuTileInfo TilesInfo { get; set; } = new ObuTileInfo(); - public bool CodedLossless { get; internal set; } + public bool CodedLossless { get; set; } - public bool[] LosslessArray { get; internal set; } = new bool[ObuConstants.MaxSegmentCount]; + public bool[] LosslessArray { get; set; } = new bool[ObuConstants.MaxSegmentCount]; public ObuQuantizationParameters QuantizationParameters { get; set; } = new ObuQuantizationParameters(); public ObuSegmentationParameters SegmentationParameters { get; set; } = new ObuSegmentationParameters(); - public bool AllLossless { get; internal set; } + public bool AllLossless { get; set; } - public bool AllowWarpedMotion { get; internal set; } + public bool AllowWarpedMotion { get; set; } - public ObuReferenceMode ReferenceMode { get; internal set; } + public ObuReferenceMode ReferenceMode { get; set; } - public ObuFilmGrainParameters FilmGrainParameters { get; internal set; } = new ObuFilmGrainParameters(); + public ObuFilmGrainParameters FilmGrainParameters { get; set; } = new ObuFilmGrainParameters(); - public bool ReducedTxSet { get; internal set; } + public bool ReducedTransformSet { get; set; } - public ObuLoopFilterParameters LoopFilterParameters { get; internal set; } = new ObuLoopFilterParameters(); + public ObuLoopFilterParameters LoopFilterParameters { get; set; } = new ObuLoopFilterParameters(); - public ObuLoopRestorationParameters[] LoopRestorationParameters { get; internal set; } = new ObuLoopRestorationParameters[3]; + public ObuLoopRestorationParameters[] LoopRestorationParameters { get; set; } = new ObuLoopRestorationParameters[3]; - public ObuConstraintDirectionalEnhancementFilterParameters ConstraintDirectionalEnhancementFilterParameters { get; internal set; } = new ObuConstraintDirectionalEnhancementFilterParameters(); + public ObuConstraintDirectionalEnhancementFilterParameters CdefParameters { get; set; } = new ObuConstraintDirectionalEnhancementFilterParameters(); - public int ModeInfoStride { get; internal set; } + public int ModeInfoStride { get; set; } - public bool DisableFrameEndUpdateCdf { get; internal set; } + public bool DisableFrameEndUpdateCdf { get; set; } + + public ObuSkipModeParameters SkipModeParameters { get; set; } = new ObuSkipModeParameters(); + + public Av1TransformMode TransformMode { get; set; } + + public ObuDeltaLoopFilterParameters DeltaLoopFilterParameters { get; set; } = new ObuDeltaLoopFilterParameters(); + + public ObuDeltaQParameters DeltaQParameters { get; set; } = new ObuDeltaQParameters(); internal ObuFrameSize FrameSize { get; set; } = new ObuFrameSize(); diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs index 961f3b2bc6..51e11610f6 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopFilterParameters.cs @@ -5,5 +5,5 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuLoopFilterParameters { - public bool[] FilterLevel { get; internal set; } + public int[] FilterLevel { get; internal set; } = new int[2]; } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs index 4cdcb26550..f540a297fa 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs @@ -5,6 +5,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuOrderHintInfo { + public bool EnableOrderHint { get; internal set; } + internal bool EnableJointCompound { get; set; } internal bool EnableReferenceFrameMotionVectors { get; set; } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index c1676febcb..87574ed059 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -693,10 +693,10 @@ internal class ObuReader // Single image is always Intra. } - SetupFrameBufferReferences(sequenceHeader, frameInfo); - CheckAddTemporalMotionVectorBuffer(sequenceHeader, frameInfo); - SetupFrameSignBias(sequenceHeader, frameInfo); + // SetupFrameBufferReferences(sequenceHeader, frameInfo); + // CheckAddTemporalMotionVectorBuffer(sequenceHeader, frameInfo); + // SetupFrameSignBias(sequenceHeader, frameInfo); if (sequenceHeader.IsReducedStillPictureHeader || frameInfo.DisableCdfUpdate) { frameInfo.DisableFrameEndUpdateCdf = true; @@ -704,22 +704,21 @@ internal class ObuReader if (frameInfo.PrimaryReferenceFrame == ObuConstants.PrimaryReferenceFrameNone) { - SetupPastIndependence(); + // SetupPastIndependence(); } - GenerateNextReferenceFrameMap(sequenceHeader, frameInfo); - + // GenerateNextReferenceFrameMap(sequenceHeader, frameInfo); frameInfo.TilesInfo = ReadTileInfo(reader, sequenceHeader, frameInfo); - frameInfo.QuantizationParameters = ReadQuantizationParameters(reader, sequenceHeader.ColorConfig, planesCount); - ReadSegmentationParameters(reader, sequenceHeader, frameInfo); - ReadDeltaQParameters(reader, frameInfo); - ReadDeltaLoopFilterParameters(reader, frameInfo); - SetupSegmentationDequantization(); + ReadQuantizationParameters(reader, frameInfo.QuantizationParameters, sequenceHeader.ColorConfig, planesCount); + ReadSegmentationParameters(reader, sequenceHeader, frameInfo, planesCount); + ReadFrameDeltaQParameters(reader, frameInfo); + ReadFrameDeltaLoopFilterParameters(reader, frameInfo); + // SetupSegmentationDequantization(); Av1MainParseContext mainParseContext = new(); if (frameInfo.PrimaryReferenceFrame == ObuConstants.PrimaryReferenceFrameNone) { - ResetParseContext(mainParseContext, frameInfo.QuantizationParameters.BaseQIndex); + // ResetParseContext(mainParseContext, frameInfo.QuantizationParameters.BaseQIndex); } int tilesCount = frameInfo.TilesInfo.TileColumnCount * frameInfo.TilesInfo.TileRowCount; @@ -758,20 +757,20 @@ internal class ObuReader frameInfo.AllLossless = frameInfo.CodedLossless && frameInfo.FrameSize.FrameWidth == frameInfo.FrameSize.SuperResolutionUpscaledWidth; ReadLoopFilterParameters(reader, sequenceHeader, frameInfo, planesCount); - ReadConstraintDirectionalEnhancementFilterParameters(reader, sequenceHeader, frameInfo, planesCount); - ReadLoopRestorationParameter(reader, sequenceHeader, frameInfo, planesCount); + ReadCdefParameters(reader, sequenceHeader, frameInfo, planesCount); + ReadLoopRestorationParameters(reader, sequenceHeader, frameInfo, planesCount); ReadTransformMode(reader, frameInfo); frameInfo.ReferenceMode = ReadFrameReferenceMode(reader, isIntraFrame); - ReadSkipModeParameters(reader, frameInfo, isIntraFrame, sequenceHeader, frameInfo.ReferenceMode); + ReadSkipModeParameters(reader, sequenceHeader, frameInfo, isIntraFrame, frameInfo.ReferenceMode); if (isIntraFrame || frameInfo.ErrorResilientMode || !sequenceHeader.EnableWarpedMotion) { frameInfo.AllowWarpedMotion = false; } - frameInfo.ReducedTxSet = reader.ReadBoolean(); + frameInfo.ReducedTransformSet = reader.ReadBoolean(); ReadGlobalMotionParameters(reader, sequenceHeader, frameInfo, isIntraFrame); - frameInfo.FilmGrainParameters = ReadFilmGrainParameters(reader, sequenceHeader, frameInfo); + frameInfo.FilmGrainParameters = ReadFilmGrainFilterParameters(reader, sequenceHeader, frameInfo); } private static bool IsSegmentationFeatureActive(ObuSegmentationParameters segmentationParameters, int segmentId, ObuSegmentationLevelFeature feature) @@ -839,11 +838,11 @@ internal class ObuReader header.PayloadSize -= headerBytes; bool noIbc = !frameInfo.AllowIntraBlockCopy; - bool doLoopFilter = noIbc && (frameInfo.LoopFilterParameters.FilterLevel[0] || frameInfo.LoopFilterParameters.FilterLevel[1]); + bool doLoopFilter = noIbc && (frameInfo.LoopFilterParameters.FilterLevel[0] != 0 || frameInfo.LoopFilterParameters.FilterLevel[1] != 0); bool doCdef = noIbc && (!frameInfo.CodedLossless && - (frameInfo.ConstraintDirectionalEnhancementFilterParameters.BitCount || - frameInfo.ConstraintDirectionalEnhancementFilterParameters.YStrength[0] != 0 || - frameInfo.ConstraintDirectionalEnhancementFilterParameters.UVStrength[0] != 0)); + (frameInfo.CdefParameters.BitCount != 0 || + frameInfo.CdefParameters.YStrength[0] != 0 || + frameInfo.CdefParameters.UVStrength[0] != 0)); bool doLoopRestoration = noIbc && (frameInfo.LoopRestorationParameters[(int)Av1Plane.Y].FrameRestorationType != ObuRestorationType.RestoreNone || frameInfo.LoopRestorationParameters[(int)Av1Plane.U].FrameRestorationType != ObuRestorationType.RestoreNone || @@ -862,7 +861,7 @@ internal class ObuReader } // TODO: Pass more info to the decoder. - DecodeTile(sequenceHeader, frameInfo, tileInfo, tileNum); + // DecodeTile(sequenceHeader, frameInfo, tileInfo, tileNum); } if (tileGroupEnd != tileCount - 1) @@ -870,7 +869,240 @@ internal class ObuReader return; } - FinishDecodeTiles(sequenceHeader, frameInfo, doCdef, doLoopRestoration); + // FinishDecodeTiles(sequenceHeader, frameInfo, doCdef, doLoopRestoration); + } + + private static int ReadDeltaQ(Av1BitStreamReader reader) + { + int deltaQ = 0; + if (reader.ReadBoolean()) + { + deltaQ = (int)reader.ReadLiteral(7); + } + + return deltaQ; + } + + private static void ReadFrameDeltaQParameters(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + { + frameInfo.DeltaQParameters.Resolution = 0; + frameInfo.DeltaQParameters.IsPresent = false; + if (frameInfo.QuantizationParameters.BaseQIndex > 0) + { + frameInfo.DeltaQParameters.IsPresent = reader.ReadBoolean(); + } + + if (frameInfo.DeltaQParameters.IsPresent) + { + frameInfo.DeltaQParameters.Resolution = (int)reader.ReadLiteral(2); + } + } + + private static void ReadFrameDeltaLoopFilterParameters(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + { + frameInfo.DeltaLoopFilterParameters.IsPresent = false; + frameInfo.DeltaLoopFilterParameters.Resolution = 0; + frameInfo.DeltaLoopFilterParameters.Multi = false; + if (frameInfo.DeltaQParameters.IsPresent) + { + if (!frameInfo.AllowIntraBlockCopy) + { + frameInfo.DeltaLoopFilterParameters.IsPresent = reader.ReadBoolean(); + } + + if (frameInfo.DeltaLoopFilterParameters.IsPresent) + { + frameInfo.DeltaLoopFilterParameters.Resolution = (int)reader.ReadLiteral(4); + frameInfo.DeltaLoopFilterParameters.Multi = reader.ReadBoolean(); + } + } + } + + private static void ReadQuantizationParameters(Av1BitStreamReader reader, ObuQuantizationParameters quantParams, ObuColorConfig colorInfo, int planesCount) + { + quantParams.BaseQIndex = (int)reader.ReadLiteral(8); + quantParams.DeltaQDc[(int)Av1Plane.Y] = ReadDeltaQ(reader); + quantParams.DeltaQAc[(int)Av1Plane.Y] = 0; + if (planesCount > 1) + { + bool areUvDeltaDifferent = false; + quantParams.DeltaQDc[(int)Av1Plane.U] = ReadDeltaQ(reader); + quantParams.DeltaQAc[(int)Av1Plane.U] = ReadDeltaQ(reader); + if (areUvDeltaDifferent) + { + quantParams.DeltaQDc[(int)Av1Plane.V] = ReadDeltaQ(reader); + quantParams.DeltaQAc[(int)Av1Plane.V] = ReadDeltaQ(reader); + } + else + { + quantParams.DeltaQDc[(int)Av1Plane.V] = quantParams.DeltaQDc[(int)Av1Plane.U]; + quantParams.DeltaQAc[(int)Av1Plane.V] = quantParams.DeltaQAc[(int)Av1Plane.U]; + } + } + else + { + quantParams.DeltaQDc[(int)Av1Plane.U] = 0; + quantParams.DeltaQAc[(int)Av1Plane.U] = 0; + quantParams.DeltaQDc[(int)Av1Plane.V] = 0; + quantParams.DeltaQAc[(int)Av1Plane.V] = 0; + } + + quantParams.IsUsingQMatrix = reader.ReadBoolean(); + if (quantParams.IsUsingQMatrix) + { + quantParams.QMatrix[(int)Av1Plane.Y] = (int)reader.ReadLiteral(4); + quantParams.QMatrix[(int)Av1Plane.U] = (int)reader.ReadLiteral(4); + if (!colorInfo.HasSeparateUvDeltaQ) + { + quantParams.QMatrix[(int)Av1Plane.V] = quantParams.QMatrix[(int)Av1Plane.U]; + } + else + { + quantParams.QMatrix[(int)Av1Plane.V] = (int)reader.ReadLiteral(4); + } + } + else + { + quantParams.QMatrix[(int)Av1Plane.Y] = 0; + quantParams.QMatrix[(int)Av1Plane.U] = 0; + quantParams.QMatrix[(int)Av1Plane.V] = 0; + } + } + + private static void ReadSegmentationParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + { + frameInfo.SegmentationParameters.SegmentationEnabled = reader.ReadBoolean(); + if (!frameInfo.SegmentationParameters.SegmentationEnabled) + { + // CopyFeatureInfo(); + return; + } + + // TODO: Parse more stuff. + } + + private static void ReadLoopFilterParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + { + if (frameInfo.CodedLossless || frameInfo.AllowIntraBlockCopy) + { + frameInfo.LoopFilterParameters.FilterLevel[0] = 0; + frameInfo.LoopFilterParameters.FilterLevel[1] = 0; + return; + } + + // TODO: Parse more stuff. + } + + private static void ReadTransformMode(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + { + if (frameInfo.CodedLossless) + { + frameInfo.TransformMode = Av1TransformMode.Only4x4; + } + else + { + if (reader.ReadBoolean()) + { + frameInfo.TransformMode = Av1TransformMode.Select; + } + else + { + frameInfo.TransformMode = Av1TransformMode.Largest; + } + } + } + + private static void ReadLoopRestorationParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + { + _ = planesCount; + if (frameInfo.CodedLossless || frameInfo.AllowIntraBlockCopy || !sequenceHeader.EnableRestoration) + { + frameInfo.LoopRestorationParameters[0].FrameRestorationType = ObuRestorationType.RestoreNone; + frameInfo.LoopRestorationParameters[1].FrameRestorationType = ObuRestorationType.RestoreNone; + frameInfo.LoopRestorationParameters[2].FrameRestorationType = ObuRestorationType.RestoreNone; + return; + } + + // TODO: Parse more stuff. + } + + private static void ReadCdefParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + { + _ = planesCount; + if (frameInfo.CodedLossless || frameInfo.AllowIntraBlockCopy || sequenceHeader.CdefLevel == 0) + { + frameInfo.CdefParameters.BitCount = 0; + frameInfo.CdefParameters.YStrength[0] = 0; + frameInfo.CdefParameters.YStrength[4] = 0; + frameInfo.CdefParameters.UVStrength[0] = 0; + frameInfo.CdefParameters.UVStrength[4] = 0; + frameInfo.CdefParameters.Damping = 0; + return; + } + + // TODO: Parse more stuff. + } + + private static void ReadGlobalMotionParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool isIntraFrame) + { + _ = reader; + _ = sequenceHeader; + _ = frameInfo; + if (isIntraFrame) + { + return; + } + + // TODO: Parse more stuff. + } + + private static ObuReferenceMode ReadFrameReferenceMode(Av1BitStreamReader reader, bool isIntraFrame) + { + if (isIntraFrame) + { + return ObuReferenceMode.SingleReference; + } + + return (ObuReferenceMode)reader.ReadLiteral(1); + } + + private static void ReadSkipModeParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool isIntraFrame, ObuReferenceMode referenceSelect) + { + if (isIntraFrame || referenceSelect == ObuReferenceMode.ReferenceModeSelect || !sequenceHeader.OrderHintInfo.EnableOrderHint) + { + frameInfo.SkipModeParameters.SkipModeAllowed = false; + } + else + { + // TODO: Parse more stuff. + } + + if (frameInfo.SkipModeParameters.SkipModeAllowed) + { + frameInfo.SkipModeParameters.SkipModeFlag = reader.ReadBoolean(); + } + else + { + frameInfo.SkipModeParameters.SkipModeFlag = false; + } + } + + private static ObuFilmGrainParameters ReadFilmGrainFilterParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + { + ObuFilmGrainParameters grainParams = new(); + if (!sequenceHeader.AreFilmGrainingParametersPresent || (!frameInfo.ShowFrame && !frameInfo.ShowableFrame)) + { + return grainParams; + } + + grainParams.ApplyGrain = reader.ReadBoolean(); + if (!grainParams.ApplyGrain) + { + return grainParams; + } + + // TODO: Implement parsing. + return grainParams; } private static bool IsValidSequenceLevel(int sequenceLevelIndex) diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSkipModeParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSkipModeParameters.cs new file mode 100644 index 0000000000..29a87a5f23 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSkipModeParameters.cs @@ -0,0 +1,11 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuSkipModeParameters +{ + public bool SkipModeAllowed { get; set; } + + public bool SkipModeFlag { get; internal set; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs index 24c8f963ea..ac5379f88a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuTileInfo.cs @@ -23,13 +23,13 @@ internal class ObuTileInfo internal int TileColumnCount { get; set; } - internal int[] TileColumnStartModeInfo { get; set; } + internal int[] TileColumnStartModeInfo { get; set; } = new int[ObuConstants.MaxTileRowCount + 1]; internal int MinLog2TileRowCount { get; set; } internal int TileRowCountLog2 { get; set; } - internal int[] TileRowStartModeInfo { get; set; } + internal int[] TileRowStartModeInfo { get; set; } = new int[ObuConstants.MaxTileColumnCount + 1]; internal int TileRowCount { get; set; } From e45a41b54e2d3754f5ad60297d19c5318414ae3b Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Fri, 12 Jan 2024 12:16:04 +0100 Subject: [PATCH 03/12] Av1 bitstream tests --- .../Formats/Heif/Av1/Av1BitStreamReader.cs | 23 ++--- .../Formats/Heif/Av1/Av1BitStreamWriter.cs | 56 ++++++++++++ .../Formats/Heif/Av1/Av1BitStreamTests.cs | 86 +++++++++++++++++++ 3 files changed, 149 insertions(+), 16 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs create mode 100644 tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs index e16553121c..8804c31626 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs @@ -1,11 +1,13 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Buffers.Binary; + namespace SixLabors.ImageSharp.Formats.Heif.Av1; internal ref struct Av1BitStreamReader(Span data) { - private const int WordSize = sizeof(byte); + private const int WordSize = sizeof(byte) * 8; private const int DoubleWordSize = 2 * WordSize; private readonly Span data = data; private int wordPosition = 0; @@ -33,25 +35,14 @@ internal ref struct Av1BitStreamReader(Span data) public uint ReadLiteral(int bitCount) { - uint bits = (uint)(this.data[this.wordPosition] << this.bitOffset) >> (WordSize - bitCount); - this.bitOffset += bitCount; - while (this.bitOffset > WordSize) - { - uint nextWord = this.data[this.wordPosition + 1]; - bits |= nextWord << (DoubleWordSize - bitCount); - } - - if (this.bitOffset >= WordSize) - { - this.bitOffset -= WordSize; - } - - return bits; + uint bits = BinaryPrimitives.ReadUInt32BigEndian(this.data[this.wordPosition..]); + this.Skip(bitCount); + return bits >> (32 - bitCount); } internal bool ReadBoolean() { - bool bit = (this.data[this.wordPosition] & (1 << (WordSize - this.bitOffset))) > 0; + bool bit = (this.data[this.wordPosition] & (1 << (WordSize - this.bitOffset - 1))) > 0; this.Skip(1); return bit; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs new file mode 100644 index 0000000000..4459c01a00 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs @@ -0,0 +1,56 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal ref struct Av1BitStreamWriter(Stream stream) +{ + private const int WordSize = sizeof(byte) * 8; + private readonly Stream stream = stream; + private byte buffer = 0; + private int bitOffset = 0; + + public readonly int BitPosition => (int)(this.stream.Position * WordSize) + this.bitOffset; + + public readonly int Length => (int)this.stream.Length; + + public void Skip(int bitCount) + { + this.bitOffset += bitCount; + while (this.bitOffset >= WordSize) + { + this.bitOffset -= WordSize; + this.stream.WriteByte(this.buffer); + this.buffer = 0; + } + } + + public void Flush() + { + this.stream.WriteByte(this.buffer); + this.bitOffset = 0; + } + + public void WriteLiteral(uint value, int bitCount) + { + int shift = 24; + uint padded = value << ((32 - bitCount) - this.bitOffset); + while (bitCount >= 8) + { + byte current = (byte)(((padded >> shift) & 0xff) | this.buffer); + this.stream.WriteByte(current); + shift -= 8; + bitCount -= 8; + this.buffer = 0; + this.bitOffset = 0; + } + + if (bitCount > 0) + { + this.buffer = (byte)(((padded >> shift) & 0xff) | this.buffer); + this.bitOffset += bitCount; + } + } + + internal void WriteBoolean(bool value) => this.WriteLiteral(value ? 1U : 0U, 1); +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs new file mode 100644 index 0000000000..7d6c75dded --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs @@ -0,0 +1,86 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers.Binary; +using SixLabors.ImageSharp.Formats.Heif.Av1; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +[Trait("Format", "Avif")] +public class Av1BitsStreamTests +{ + [Theory] + [InlineData(42, new bool[] { false, false, true, false, true, false, true, false })] + [InlineData(52, new bool[] { false, false, true, true, false, true, false, false })] + public void ReadAsBoolean(int value, bool[] bits) + { + int bitCount = bits.Length; + byte[] buffer = new byte[4]; + BinaryPrimitives.WriteInt32BigEndian(buffer, value << (32 - bitCount)); + Av1BitStreamReader reader = new(buffer); + bool[] actual = new bool[bitCount]; + for (int i = 0; i < bitCount; i++) + { + actual[i] = reader.ReadBoolean(); + } + + Assert.Equal(bits, actual); + } + + [Theory] + [InlineData(6, 4)] + [InlineData(42, 8)] + [InlineData(52, 8)] + [InlineData(4050, 16)] + public void ReadAsLiteral(uint expected, int bitCount) + { + byte[] buffer = new byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(buffer, expected << (32 - bitCount)); + Av1BitStreamReader reader = new(buffer); + uint actual = reader.ReadLiteral(bitCount); + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData(new bool[] { false, false, true, false, true, false, true, false })] + [InlineData(new bool[] { false, true, false, true })] + public void WriteAsBoolean(bool[] booleans) + { + using MemoryStream stream = new(4); + Av1BitStreamWriter writer = new(stream); + for (int i = 0; i < booleans.Length; i++) + { + writer.WriteBoolean(booleans[i]); + } + + writer.Flush(); + + // Read the written value back. + Av1BitStreamReader reader = new(stream.GetBuffer()); + bool[] actual = new bool[booleans.Length]; + for (int i = 0; i < booleans.Length; i++) + { + actual[i] = reader.ReadBoolean(); + } + + Assert.Equal(booleans, actual); + } + + [Theory] + [InlineData(6, 4)] + [InlineData(42, 8)] + [InlineData(52, 8)] + [InlineData(4050, 16)] + public void WriteAsLiteral(uint value, int bitCount) + { + using MemoryStream stream = new(4); + Av1BitStreamWriter writer = new(stream); + writer.WriteLiteral(value, bitCount); + writer.Flush(); + + // Read the written value back. + Av1BitStreamReader reader = new(stream.GetBuffer()); + uint actual = reader.ReadLiteral(bitCount); + Assert.Equal(value, actual); + } +} From 7b961ff3296c6290a4544a82d163a573b7bd2423 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Fri, 12 Jan 2024 12:29:40 +0100 Subject: [PATCH 04/12] Optimized WriteBoolean() --- .../Formats/Heif/Av1/Av1BitStreamReader.cs | 1 - .../Formats/Heif/Av1/Av1BitStreamWriter.cs | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs index 8804c31626..e1f80b1af4 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs @@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1; internal ref struct Av1BitStreamReader(Span data) { private const int WordSize = sizeof(byte) * 8; - private const int DoubleWordSize = 2 * WordSize; private readonly Span data = data; private int wordPosition = 0; private int bitOffset = 0; diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs index 4459c01a00..a5c55530ec 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs @@ -5,7 +5,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1; internal ref struct Av1BitStreamWriter(Stream stream) { - private const int WordSize = sizeof(byte) * 8; + private const int WordSize = 8; private readonly Stream stream = stream; private byte buffer = 0; private int bitOffset = 0; @@ -35,7 +35,7 @@ internal ref struct Av1BitStreamWriter(Stream stream) { int shift = 24; uint padded = value << ((32 - bitCount) - this.bitOffset); - while (bitCount >= 8) + while ((bitCount + this.bitOffset) >= 8) { byte current = (byte)(((padded >> shift) & 0xff) | this.buffer); this.stream.WriteByte(current); @@ -52,5 +52,15 @@ internal ref struct Av1BitStreamWriter(Stream stream) } } - internal void WriteBoolean(bool value) => this.WriteLiteral(value ? 1U : 0U, 1); + internal void WriteBoolean(bool value) + { + byte boolByte = value ? (byte)1 : (byte)0; + this.buffer = (byte)(((boolByte << (7 - this.bitOffset)) & 0xff) | this.buffer); + this.bitOffset++; + if (this.bitOffset == WordSize) + { + this.stream.WriteByte(this.buffer); + this.bitOffset = 0; + } + } } From 961d55e9be49ae6b402e45c539e0efb0c6ae812d Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Fri, 19 Jan 2024 14:27:19 +0100 Subject: [PATCH 05/12] Skeleton --- .../Formats/Heif/Av1/Av1BitDepth.cs | 11 ++ .../Formats/Heif/Av1/Av1BlockModeInfo.cs | 16 ++ src/ImageSharp/Formats/Heif/Av1/Av1Math.cs | 15 ++ .../Formats/Heif/Av1/Av1PartitionType.cs | 62 +++++++ .../Av1/OpenBitstreamUnit/ObuConstants.cs | 2 +- .../Av1/OpenBitstreamUnit/ObuFrameHeader.cs | 2 + .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 2 + .../Heif/Av1/Prediction/Av1PredictionMode.cs | 26 +++ .../Heif/Av1/Transform/Av1InverseQuantizer.cs | 16 ++ .../Heif/Av1/Transform/Av1Quantization.cs | 152 ++++++++++++++++++ .../Av1/{ => Transform}/Av1TransformMode.cs | 2 +- .../Heif/Av1/Transform/Av1TransformType.cs | 49 ++++++ src/ImageSharp/Formats/Heif/Heif4CharCode.cs | 9 +- src/ImageSharp/Formats/Heif/Heif4CharCode.tt | 5 +- .../Formats/Heif/HeifDecoderCore.cs | 2 + src/ImageSharp/ImageSharp.csproj | 4 + 16 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1BitDepth.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1PartitionType.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs rename src/ImageSharp/Formats/Heif/Av1/{ => Transform}/Av1TransformMode.cs (74%) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformType.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitDepth.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitDepth.cs new file mode 100644 index 0000000000..28d48b13cf --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitDepth.cs @@ -0,0 +1,11 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal enum Av1BitDepth : int +{ + EightBit = 0, + TenBit = 1, + TwelveBit = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs new file mode 100644 index 0000000000..43325d539c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs @@ -0,0 +1,16 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal class Av1BlockModeInfo +{ + public Av1BlockSize BlockSize { get; } + + public Av1PredictionMode PredictionMode { get; } + + public Av1PartitionType Partition { get; } + +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs index c98652e048..36929e3c65 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs @@ -77,6 +77,21 @@ internal static class Av1Math return z; } + public static int Clip3(int x, int y, int z) + { + if (z < x) + { + return x; + } + + if (z > y) + { + return y; + } + + return z; + } + public static uint Round2(uint value, int n) { if (n == 0) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1PartitionType.cs b/src/ImageSharp/Formats/Heif/Av1/Av1PartitionType.cs new file mode 100644 index 0000000000..e5aa1f5b18 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1PartitionType.cs @@ -0,0 +1,62 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal enum Av1PartitionType +{ + /// + /// Not partitioned any further. + /// + None, + + /// + /// Horizontally split in 2 partitions. + /// + Horizontal, + + /// + /// Vertically split in 2 partitions. + /// + Vertical, + + /// + /// 4 equally sized partitions. + /// + Split, + + /// + /// Horizontal split and the top partition is split again. + /// + HorizontalA, + + /// + /// Horizontal split and the bottom partition is split again. + /// + HorizontalB, + + /// + /// Vertical split and the left partition is split again. + /// + VerticalA, + + /// + /// Vertical split and the right partitino is split again. + /// + VerticalB, + + /// + /// 4:1 horizontal partition. + /// + Horizontal4, + + /// + /// 4:1 vertical partition. + /// + Vertical4, + + /// + /// Invalid value. + /// + Invalid = 255 +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs index 678b49d8b8..392a54a5c0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstants.cs @@ -7,7 +7,7 @@ internal static class ObuConstants { public const ObuSequenceProfile MaxSequenceProfile = ObuSequenceProfile.Professional; - public const int LevelBits = -1; + public const int LevelBits = 5; /// /// Number of fractional bits for computing position in upscaling. diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs index 3ccf43a36d..315f0f18f2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuFrameHeader diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index 87574ed059..b9a6425779 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuReader diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs new file mode 100644 index 0000000000..a46733b007 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs @@ -0,0 +1,26 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +// Inter modes are not defined here, as they do not apply to pictures. +internal enum Av1PredictionMode +{ + DC, + Vertical, + Horizontal, + Directional45Degrees, + Directional135Degrees, + Directional113Degrees, + Directional157Degrees, + Directional203Degrees, + Directional67Degrees, + Smooth, + SmoothVertical, + SmoothHorizontal, + Paeth, + IntraModeStart = DC, + IntraModeEnd = Paeth + 1, + IntraModes = Paeth, + IntraInvalid = 25, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs new file mode 100644 index 0000000000..8d66221785 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs @@ -0,0 +1,16 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +internal class Av1InverseQuantizer +{ + + public int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) + { + return 0; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs new file mode 100644 index 0000000000..205ffaaa89 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs @@ -0,0 +1,152 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +internal static class Av1Quantization +{ + public static readonly int[,] AcQLookup = new int[3, 256] + { + { + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, + 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, + 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, 227, 231, 235, 239, 243, 247, 251, 255, 260, + 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, + 373, 380, 387, 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, + 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, 702, 715, 729, 743, + 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, + 1087, 1108, 1129, 1151, 1173, 1196, 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, + 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, + }, + { + 4, 9, 11, 13, 16, 18, 21, 24, 27, 30, 33, 37, 40, 44, 48, 51, 55, 59, 63, + 67, 71, 75, 79, 83, 88, 92, 96, 100, 105, 109, 114, 118, 122, 127, 131, 136, 140, 145, + 149, 154, 158, 163, 168, 172, 177, 181, 186, 190, 195, 199, 204, 208, 213, 217, 222, 226, 231, + 235, 240, 244, 249, 253, 258, 262, 267, 271, 275, 280, 284, 289, 293, 297, 302, 306, 311, 315, + 319, 324, 328, 332, 337, 341, 345, 349, 354, 358, 362, 367, 371, 375, 379, 384, 388, 392, 396, + 401, 409, 417, 425, 433, 441, 449, 458, 466, 474, 482, 490, 498, 506, 514, 523, 531, 539, 547, + 555, 563, 571, 579, 588, 596, 604, 616, 628, 640, 652, 664, 676, 688, 700, 713, 725, 737, 749, + 761, 773, 785, 797, 809, 825, 841, 857, 873, 889, 905, 922, 938, 954, 970, 986, 1002, 1018, 1038, + 1058, 1078, 1098, 1118, 1138, 1158, 1178, 1198, 1218, 1242, 1266, 1290, 1314, 1338, 1362, 1386, 1411, 1435, 1463, + 1491, 1519, 1547, 1575, 1603, 1631, 1663, 1695, 1727, 1759, 1791, 1823, 1859, 1895, 1931, 1967, 2003, 2039, 2079, + 2119, 2159, 2199, 2239, 2283, 2327, 2371, 2415, 2459, 2507, 2555, 2603, 2651, 2703, 2755, 2807, 2859, 2915, 2971, + 3027, 3083, 3143, 3203, 3263, 3327, 3391, 3455, 3523, 3591, 3659, 3731, 3803, 3876, 3952, 4028, 4104, 4184, 4264, + 4348, 4432, 4516, 4604, 4692, 4784, 4876, 4972, 5068, 5168, 5268, 5372, 5476, 5584, 5692, 5804, 5916, 6032, 6148, + 6268, 6388, 6512, 6640, 6768, 6900, 7036, 7172, 7312, + }, + { + 4, 13, 19, 27, 35, 44, 54, 64, 75, 87, 99, 112, 126, 139, 154, 168, + 183, 199, 214, 230, 247, 263, 280, 297, 314, 331, 349, 366, 384, 402, 420, 438, + 456, 475, 493, 511, 530, 548, 567, 586, 604, 623, 642, 660, 679, 698, 716, 735, + 753, 772, 791, 809, 828, 846, 865, 884, 902, 920, 939, 957, 976, 994, 1012, 1030, + 1049, 1067, 1085, 1103, 1121, 1139, 1157, 1175, 1193, 1211, 1229, 1246, 1264, 1282, 1299, 1317, + 1335, 1352, 1370, 1387, 1405, 1422, 1440, 1457, 1474, 1491, 1509, 1526, 1543, 1560, 1577, 1595, + 1627, 1660, 1693, 1725, 1758, 1791, 1824, 1856, 1889, 1922, 1954, 1987, 2020, 2052, 2085, 2118, + 2150, 2183, 2216, 2248, 2281, 2313, 2346, 2378, 2411, 2459, 2508, 2556, 2605, 2653, 2701, 2750, + 2798, 2847, 2895, 2943, 2992, 3040, 3088, 3137, 3185, 3234, 3298, 3362, 3426, 3491, 3555, 3619, + 3684, 3748, 3812, 3876, 3941, 4005, 4069, 4149, 4230, 4310, 4390, 4470, 4550, 4631, 4711, 4791, + 4871, 4967, 5064, 5160, 5256, 5352, 5448, 5544, 5641, 5737, 5849, 5961, 6073, 6185, 6297, 6410, + 6522, 6650, 6778, 6906, 7034, 7162, 7290, 7435, 7579, 7723, 7867, 8011, 8155, 8315, 8475, 8635, + 8795, 8956, 9132, 9308, 9484, 9660, 9836, 10028, 10220, 10412, 10604, 10812, 11020, 11228, 11437, 11661, + 11885, 12109, 12333, 12573, 12813, 13053, 13309, 13565, 13821, 14093, 14365, 14637, 14925, 15213, 15502, 15806, + 16110, 16414, 16734, 17054, 17390, 17726, 18062, 18414, 18766, 19134, 19502, 19886, 20270, 20670, 21070, 21486, + 21902, 22334, 22766, 23214, 23662, 24126, 24590, 25070, 25551, 26047, 26559, 27071, 27599, 28143, 28687, 29247, + } + }; + + private static readonly int[,] DcQLookup = new int[3, 256] + { + { + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, + 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, + 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, 73, + 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, + 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, 116, 117, 118, 120, 121, + 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, + 166, 169, 172, 174, 177, 180, 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, + 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, 284, 288, 292, + 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, + 389, 395, 400, 406, 411, 417, 423, 429, 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, + 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, 775, + 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, + }, + { + 4, 9, 10, 13, 15, 17, 20, 22, 25, 28, 31, 34, 37, 40, 43, 47, 50, 53, 57, + 60, 64, 68, 71, 75, 78, 82, 86, 90, 93, 97, 101, 105, 109, 113, 116, 120, 124, 128, + 132, 136, 140, 143, 147, 151, 155, 159, 163, 166, 170, 174, 178, 182, 185, 189, 193, 197, 200, + 204, 208, 212, 215, 219, 223, 226, 230, 233, 237, 241, 244, 248, 251, 255, 259, 262, 266, 269, + 273, 276, 280, 283, 287, 290, 293, 297, 300, 304, 307, 310, 314, 317, 321, 324, 327, 331, 334, + 337, 343, 350, 356, 362, 369, 375, 381, 387, 394, 400, 406, 412, 418, 424, 430, 436, 442, 448, + 454, 460, 466, 472, 478, 484, 490, 499, 507, 516, 525, 533, 542, 550, 559, 567, 576, 584, 592, + 601, 609, 617, 625, 634, 644, 655, 666, 676, 687, 698, 708, 718, 729, 739, 749, 759, 770, 782, + 795, 807, 819, 831, 844, 856, 868, 880, 891, 906, 920, 933, 947, 961, 975, 988, 1001, 1015, 1030, + 1045, 1061, 1076, 1090, 1105, 1120, 1137, 1153, 1170, 1186, 1202, 1218, 1236, 1253, 1271, 1288, 1306, 1323, 1342, + 1361, 1379, 1398, 1416, 1436, 1456, 1476, 1496, 1516, 1537, 1559, 1580, 1601, 1624, 1647, 1670, 1692, 1717, 1741, + 1766, 1791, 1817, 1844, 1871, 1900, 1929, 1958, 1990, 2021, 2054, 2088, 2123, 2159, 2197, 2236, 2276, 2319, 2363, + 2410, 2458, 2508, 2561, 2616, 2675, 2737, 2802, 2871, 2944, 3020, 3102, 3188, 3280, 3375, 3478, 3586, 3702, 3823, + 3953, 4089, 4236, 4394, 4559, 4737, 4929, 5130, 5347, + }, + { + 4, 12, 18, 25, 33, 41, 50, 60, 70, 80, 91, 103, 115, 127, 140, 153, + 166, 180, 194, 208, 222, 237, 251, 266, 281, 296, 312, 327, 343, 358, 374, 390, + 405, 421, 437, 453, 469, 484, 500, 516, 532, 548, 564, 580, 596, 611, 627, 643, + 659, 674, 690, 706, 721, 737, 752, 768, 783, 798, 814, 829, 844, 859, 874, 889, + 904, 919, 934, 949, 964, 978, 993, 1008, 1022, 1037, 1051, 1065, 1080, 1094, 1108, 1122, + 1136, 1151, 1165, 1179, 1192, 1206, 1220, 1234, 1248, 1261, 1275, 1288, 1302, 1315, 1329, 1342, + 1368, 1393, 1419, 1444, 1469, 1494, 1519, 1544, 1569, 1594, 1618, 1643, 1668, 1692, 1717, 1741, + 1765, 1789, 1814, 1838, 1862, 1885, 1909, 1933, 1957, 1992, 2027, 2061, 2096, 2130, 2165, 2199, + 2233, 2267, 2300, 2334, 2367, 2400, 2434, 2467, 2499, 2532, 2575, 2618, 2661, 2704, 2746, 2788, + 2830, 2872, 2913, 2954, 2995, 3036, 3076, 3127, 3177, 3226, 3275, 3324, 3373, 3421, 3469, 3517, + 3565, 3621, 3677, 3733, 3788, 3843, 3897, 3951, 4005, 4058, 4119, 4181, 4241, 4301, 4361, 4420, + 4479, 4546, 4612, 4677, 4742, 4807, 4871, 4942, 5013, 5083, 5153, 5222, 5291, 5367, 5442, 5517, + 5591, 5665, 5745, 5825, 5905, 5984, 6063, 6149, 6234, 6319, 6404, 6495, 6587, 6678, 6769, 6867, + 6966, 7064, 7163, 7269, 7376, 7483, 7599, 7715, 7832, 7958, 8085, 8214, 8352, 8492, 8635, 8788, + 8945, 9104, 9275, 9450, 9639, 9832, 10031, 10245, 10465, 10702, 10946, 11210, 11482, 11776, 12081, 12409, + 12750, 13118, 13501, 13913, 14343, 14807, 15290, 15812, 16356, 16943, 17575, 18237, 18949, 19718, 20521, 21387, + } + }; + + public static int GetDcQuantization(int qIndex, int delta, Av1BitDepth bitDepth) + => DcQLookup[(int)bitDepth, Av1Math.Clip3(0, 255, qIndex + delta)]; + + public static int GetAcQuantization(int qIndex, int delta, Av1BitDepth bitDepth) + => AcQLookup[(int)bitDepth, Av1Math.Clip3(0, 255, qIndex + delta)]; + + public static int GetQzbinFactor(int q, Av1BitDepth bitDepth) + { + int quant = GetDcQuantization(q, 0, bitDepth); + switch (bitDepth) + { + case Av1BitDepth.EightBit: + return q == 0 ? 64 : (quant < 148 ? 84 : 80); + case Av1BitDepth.TenBit: + return q == 0 ? 64 : (quant < 592 ? 84 : 80); + case Av1BitDepth.TwelveBit: + return q == 0 ? 64 : (quant < 2368 ? 84 : 80); + default: + DebugGuard.IsTrue(false, "bit_depth should be EIGHT_BIT, TEN_BIT or TWELVE_BIT"); + return -1; + } + } + + public static void InvertQuantization(out int quantization, out int shift, int d) + { + uint t; + int l, m; + t = (uint)d; + for (l = 0; t > 1; l++) + { + t >>= 1; + } + + m = 1 + ((1 << (16 + l)) / d); + quantization = m - (1 << 16); + shift = 1 << (16 - l); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformMode.cs similarity index 74% rename from src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformMode.cs index 99d47f61e2..a838dfc86b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1TransformMode.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformMode.cs @@ -1,7 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -namespace SixLabors.ImageSharp.Formats.Heif.Av1; +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; internal enum Av1TransformMode : byte { diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformType.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformType.cs new file mode 100644 index 0000000000..93de8f2271 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformType.cs @@ -0,0 +1,49 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +internal enum Av1TransformType : byte +{ + /// + /// DCT in both horizontal and vertical. + /// + DctDct, + + /// + /// ADST in vertical, DCT in horizontal. + /// + AdstDct, + + /// + /// DCT in vertical, ADST in horizontal. + /// + DctAdst, + + /// + /// ADST in both directions. + /// + AdstAdst, + FlipAdstDct, + DctFlipAdst, + FlipAdstFlipAdst, + AdstFlipAdst, + FlipAdstAdst, + Identity, + VerticalDct, + HorizontalDct, + VerticalAdst, + HorizontalAdst, + VerticalFlipAdst, + HorizontalFlipAdst, + + /// + /// Number of Transform types. + /// + TransformTypes, + + /// + /// Invalid value. + /// + Invalid, +} diff --git a/src/ImageSharp/Formats/Heif/Heif4CharCode.cs b/src/ImageSharp/Formats/Heif/Heif4CharCode.cs index d6d31ef1d3..3c238f2acf 100644 --- a/src/ImageSharp/Formats/Heif/Heif4CharCode.cs +++ b/src/ImageSharp/Formats/Heif/Heif4CharCode.cs @@ -249,12 +249,17 @@ public enum Heif4CharCode : uint free = 0x66726565U, /// - /// ICC Color Profile. + /// Color profile. + /// + nclx = 0x6E636C78U, + + /// + /// ICC Color profile. /// rICC = 0x72494343U, /// - /// ICC Color Profile. + /// ICC Color profile. /// prof = 0x70726F66U, diff --git a/src/ImageSharp/Formats/Heif/Heif4CharCode.tt b/src/ImageSharp/Formats/Heif/Heif4CharCode.tt index 9f8555f33b..6c9bd514ca 100644 --- a/src/ImageSharp/Formats/Heif/Heif4CharCode.tt +++ b/src/ImageSharp/Formats/Heif/Heif4CharCode.tt @@ -54,8 +54,9 @@ "pict", "Picture handler type", "uuid", "Unique Identifier", "free", "Free space", - "rICC", "ICC Color Profile", - "prof", "ICC Color Profile", + "nclx", "Color profile", + "rICC", "ICC Color profile", + "prof", "ICC Color profile", }; #> diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index 8a8c8e29f6..b39f24517d 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -4,6 +4,8 @@ using System.Buffers; using System.Buffers.Binary; using System.Text; +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata; diff --git a/src/ImageSharp/ImageSharp.csproj b/src/ImageSharp/ImageSharp.csproj index 63bc8131bd..b76a139019 100644 --- a/src/ImageSharp/ImageSharp.csproj +++ b/src/ImageSharp/ImageSharp.csproj @@ -241,5 +241,9 @@ + + + + From 790207f40e6bea40082db990ee7a643a34652e2e Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sat, 20 Jan 2024 12:37:52 +0100 Subject: [PATCH 06/12] Prepare quantization --- .../Formats/Heif/Av1/Av1BlockModeInfo.cs | 1 - .../Av1/OpenBitstreamUnit/ObuPartitionInfo.cs | 8 +++++ .../Heif/Av1/Transform/Av1InverseQuantizer.cs | 3 +- .../Heif/Av1/Transform/Av1TransformSize.cs | 30 +++++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuPartitionInfo.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSize.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs index 43325d539c..a0db9473d4 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs @@ -12,5 +12,4 @@ internal class Av1BlockModeInfo public Av1PredictionMode PredictionMode { get; } public Av1PartitionType Partition { get; } - } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuPartitionInfo.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuPartitionInfo.cs new file mode 100644 index 0000000000..35afda11f9 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuPartitionInfo.cs @@ -0,0 +1,8 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +internal class ObuPartitionInfo +{ +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs index 8d66221785..98443393a9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs @@ -8,8 +8,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; internal class Av1InverseQuantizer { - - public int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) + public static int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) { return 0; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSize.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSize.cs new file mode 100644 index 0000000000..267c55266d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSize.cs @@ -0,0 +1,30 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +internal enum Av1TransformSize : byte +{ + Size4x4, + Size8x8, + Size16x16, + Size32x32, + Size64x64, + Size4x8, + Size8x4, + Size8x16, + Size16x8, + Size16x32, + Size32x16, + Size32x64, + Size64x32, + Size4x16, + Size16x4, + Size8x32, + Size32x8, + Size16x64, + Size64x16, + AllSizes, + SquareSizes = Size4x8, + Invalid = 255, +} From 80140c4cefa9d6e43d41dff6d587afc1a0d9ca13 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sat, 6 Apr 2024 10:18:46 +0200 Subject: [PATCH 07/12] Inverse quantization --- .../Formats/Heif/Av1/Av1BlockModeInfo.cs | 2 + .../Heif/Av1/Transform/Av1InverseQuantizer.cs | 62 ++++++++++++++- ...Quantization.cs => Av1InverseTransform.cs} | 22 +++--- .../Heif/Av1/Transform/Av1ScanOrder.cs | 27 +++++++ .../Av1/Transform/Av1ScanOrderConstants.cs | 75 +++++++++++++++++++ .../Transform/Av1TransformSizeExtensions.cs | 16 ++++ .../Formats/Heif/HeifDecoderCore.cs | 1 - 7 files changed, 189 insertions(+), 16 deletions(-) rename src/ImageSharp/Formats/Heif/Av1/Transform/{Av1Quantization.cs => Av1InverseTransform.cs} (95%) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs index a0db9473d4..c288919cec 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs @@ -12,4 +12,6 @@ internal class Av1BlockModeInfo public Av1PredictionMode PredictionMode { get; } public Av1PartitionType Partition { get; } + + public int SegmentId { get; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs index 98443393a9..b52473dbd9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs @@ -8,8 +8,66 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; internal class Av1InverseQuantizer { - public static int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) + public static int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] level, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) { - return 0; + Av1ScanOrder scanOrder = Av1ScanOrderConstants.GetScanOrder(txSize, txType); + short[] scanIndices = scanOrder.Scan; + int maxValue = (1 << (7 + sequenceHeader.ColorConfig.BitDepth)) - 1; + int minValue = -(1 << (7 + sequenceHeader.ColorConfig.BitDepth)); + bool usingQuantizationMatrix = frameHeader.QuantizationParameters.IsUsingQMatrix; + bool lossless = frameHeader.LosslessArray[mode.SegmentId]; + short[] dequant = []; // Get from DecModCtx + int qmLevel = (lossless || !usingQuantizationMatrix) ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1 : frameHeader.QuantizationParameters.QMatrix[(int)plane]; + byte[] iqMatrix = []; // txType.Is2dTransform() ? Get from DecModCtx + int shift = txSize.GetScale(); + + int coefficientCount = level[0]; + Span levelSpan = level.AsSpan(1); + int lev = levelSpan[0]; + int qCoefficient; + if (lev != 0) + { + int pos = scanIndices[0]; + qCoefficient = (int)(((long)Math.Abs(lev) * GetDequantizationValue(dequant[0], pos, iqMatrix)) & 0xffffff); + qCoefficient >>= shift; + + if (lev < 0) + { + qCoefficient = -qCoefficient; + } + + qCoefficients[0] = Av1Math.Clamp(qCoefficient, minValue, maxValue); + } + + for (int i = 1; i < coefficientCount; i++) + { + lev = levelSpan[i]; + if (lev != 0) + { + int pos = scanIndices[i]; + qCoefficient = (int)(((long)Math.Abs(lev) * GetDequantizationValue(dequant[1], pos, iqMatrix)) & 0xffffff); + qCoefficient >>= shift; + + if (lev < 0) + { + qCoefficient = -qCoefficient; + } + + qCoefficients[pos] = Av1Math.Clamp(qCoefficient, minValue, maxValue); + } + } + + return coefficientCount; + } + + private static int GetDequantizationValue(short dequant, int coefficientIndex, byte[] iqMatrix) + { + int dqv = dequant; + if (iqMatrix != null) + { + dqv = ((iqMatrix[coefficientIndex] * dqv) + (1 << (Av1ScanOrderConstants.QuantizationMatrixLevelBitCount - 1))) >> Av1ScanOrderConstants.QuantizationMatrixLevelBitCount; + } + + return dqv; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransform.cs similarity index 95% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransform.cs index 205ffaaa89..4856c47194 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransform.cs @@ -3,7 +3,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; -internal static class Av1Quantization +internal static class Av1InverseTransform { public static readonly int[,] AcQLookup = new int[3, 256] { @@ -121,18 +121,14 @@ internal static class Av1Quantization public static int GetQzbinFactor(int q, Av1BitDepth bitDepth) { int quant = GetDcQuantization(q, 0, bitDepth); - switch (bitDepth) - { - case Av1BitDepth.EightBit: - return q == 0 ? 64 : (quant < 148 ? 84 : 80); - case Av1BitDepth.TenBit: - return q == 0 ? 64 : (quant < 592 ? 84 : 80); - case Av1BitDepth.TwelveBit: - return q == 0 ? 64 : (quant < 2368 ? 84 : 80); - default: - DebugGuard.IsTrue(false, "bit_depth should be EIGHT_BIT, TEN_BIT or TWELVE_BIT"); - return -1; - } + + // Bit hack to get to: + // EightBit => 148 + // TenBit => 592 + // TwelveBit => 2368 + int shift = (int)bitDepth << 1; + int threshold = (1 << shift) * 148; + return q == 0 ? 64 : (quant < threshold ? 84 : 80); } public static void InvertQuantization(out int quantization, out int shift, int d) diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs new file mode 100644 index 0000000000..a773b5479b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs @@ -0,0 +1,27 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +internal readonly struct Av1ScanOrder +{ + public Av1ScanOrder(short[]? scan) + { + this.Scan = scan ?? []; + this.IScan = []; + this.Neighbors = []; + } + + public Av1ScanOrder(short[] scan, short[] iscan, short[] neighbors) + { + this.Scan = scan; + this.IScan = iscan; + this.Neighbors = neighbors; + } + + public short[] Scan { get; } + + public short[] IScan { get; } + + public short[] Neighbors { get; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs new file mode 100644 index 0000000000..a014ad2245 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs @@ -0,0 +1,75 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +internal static class Av1ScanOrderConstants +{ + public const int QuantizationMatrixLevelBitCount = 4; + public const int QuantizationMatrixLevelCount = 1 << QuantizationMatrixLevelBitCount; + + private static readonly Av1ScanOrder[][] ScanOrders = + [ + + // Transform size 4x4 + [ + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(MatrixRowScan4x4), + new Av1ScanOrder(MatrixColumnScan4x4), + new Av1ScanOrder(MatrixRowScan4x4), + new Av1ScanOrder(MatrixColumnScan4x4), + new Av1ScanOrder(MatrixRowScan4x4), + new Av1ScanOrder(MatrixColumnScan4x4), + ], + + // Transform size 8x8 + [ + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(MatrixRowScan8x8), + new Av1ScanOrder(MatrixColumnScan8x8), + new Av1ScanOrder(MatrixRowScan8x8), + new Av1ScanOrder(MatrixColumnScan8x8), + new Av1ScanOrder(MatrixRowScan8x8), + new Av1ScanOrder(MatrixColumnScan8x8), + ], + ]; + + private static readonly short[] DefaultScan4x4 = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; + private static readonly short[] MatrixColumnScan4x4 = [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]; + private static readonly short[] MatrixRowScan4x4 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + + private static readonly short[] DefaultScan8x8 = [0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, + 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, + 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63]; + + private static readonly short[] MatrixColumnScan8x8 = [0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57, 2, 10, 18, 26, 34, 42, + 50, 58, 3, 11, 19, 27, 35, 43, 51, 59, 4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, + 37, 45, 53, 61, 6, 14, 22, 30, 38, 46, 54, 62, 7, 15, 23, 31, 39, 47, 55, 63]; + + private static readonly short[] MatrixRowScan8x8 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]; + + public static Av1ScanOrder GetScanOrder(Av1TransformSize txSize, Av1TransformMode txMode) + => ScanOrders[(int)txSize][(int)txMode]; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs new file mode 100644 index 0000000000..3db631e0b3 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs @@ -0,0 +1,16 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +internal static class Av1TransformSizeExtensions +{ + private static readonly int[] Size2d = [ + 16, 64, 256, 1024, 4096, 32, 32, 128, 128, 512, 512, 2048, 2048, 64, 64, 256, 256, 1024, 1024]; + + public static int GetScale(this Av1TransformSize size) + { + int pels = Size2d[(int)size]; + return (pels > 1024) ? 2 : (pels > 256) ? 1 : 0; + } +} diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index c76f7bac3f..7aeeac56b4 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -6,7 +6,6 @@ using System.Buffers.Binary; using System.Text; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -using SixLabors.ImageSharp.Formats.Webp; using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata; From 40e9d682e0152f639ec7b0594159c749e8d72553 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Wed, 17 Apr 2024 23:13:10 +0200 Subject: [PATCH 08/12] Fix reading of larger literals --- .../Formats/Heif/Av1/Av1BitStreamReader.cs | 66 ++++++++++++++----- .../Formats/Heif/Av1/Av1BitStreamTests.cs | 12 ++-- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs index e1f80b1af4..b355a9a5e9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs @@ -1,19 +1,35 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System; using System.Buffers.Binary; +using System.Runtime.InteropServices; namespace SixLabors.ImageSharp.Formats.Heif.Av1; -internal ref struct Av1BitStreamReader(Span data) +internal ref struct Av1BitStreamReader { - private const int WordSize = sizeof(byte) * 8; - private readonly Span data = data; + private const int WordSize = 32; + + private readonly Span data; + private uint currentWord; + private uint nextWord; private int wordPosition = 0; private int bitOffset = 0; - public readonly int BitPosition => (this.wordPosition * WordSize) + this.bitOffset; + public Av1BitStreamReader(Span data) + { + this.data = MemoryMarshal.Cast(data); + this.wordPosition = -1; + this.AdvanceToNextWord(); + this.AdvanceToNextWord(); + } + public readonly int BitPosition => ((this.wordPosition - 1) * WordSize) + this.bitOffset; + + /// + /// Gets the number of bytes in the readers buffer. + /// public readonly int Length => this.data.Length; public void Reset() @@ -34,22 +50,34 @@ internal ref struct Av1BitStreamReader(Span data) public uint ReadLiteral(int bitCount) { - uint bits = BinaryPrimitives.ReadUInt32BigEndian(this.data[this.wordPosition..]); - this.Skip(bitCount); - return bits >> (32 - bitCount); - } + DebugGuard.MustBeBetweenOrEqualTo(bitCount, 1, 32, nameof(bitCount)); - internal bool ReadBoolean() - { - bool bit = (this.data[this.wordPosition] & (1 << (WordSize - this.bitOffset - 1))) > 0; - this.Skip(1); - return bit; + uint bits = (this.currentWord << this.bitOffset) >> (WordSize - bitCount); + this.bitOffset += bitCount; + if (this.bitOffset > WordSize) + { + int overshoot = WordSize + WordSize - this.bitOffset; + if (overshoot < 32) + { + bits |= this.nextWord >> overshoot; + } + } + + if (this.bitOffset >= WordSize) + { + this.AdvanceToNextWord(); + this.bitOffset -= WordSize; + } + + return bits; } + internal bool ReadBoolean() => this.ReadLiteral(1) > 0; + public ulong ReadLittleEndianBytes128(out int length) { // See section 4.10.5 of the AV1-Specification - DebugGuard.IsTrue((this.bitOffset & (WordSize - 1)) == 0, "Reading of Little Endian 128 value only allowed on byte alignment"); + DebugGuard.IsTrue((this.bitOffset & 0xF7) == 0, "Reading of Little Endian 128 value only allowed on byte alignment"); ulong value = 0; length = 0; @@ -58,7 +86,7 @@ internal ref struct Av1BitStreamReader(Span data) uint leb128Byte = this.ReadLiteral(8); value |= (leb128Byte & 0x7FUL) << i; length++; - if ((leb128Byte & 0x80U) != 0x80U) + if ((leb128Byte & 0x80U) == 0) { break; } @@ -137,4 +165,12 @@ internal ref struct Av1BitStreamReader(Span data) return t; } + + public void AdvanceToNextWord() + { + this.currentWord = this.nextWord; + this.wordPosition++; + uint temp = this.data[this.wordPosition]; + this.nextWord = (temp << 24) | ((temp & 0x0000ff00U) << 8) | ((temp & 0x00ff0000U) >> 8) | (temp >> 24); + } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs index 7d6c75dded..263182a6dd 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs @@ -12,11 +12,11 @@ public class Av1BitsStreamTests [Theory] [InlineData(42, new bool[] { false, false, true, false, true, false, true, false })] [InlineData(52, new bool[] { false, false, true, true, false, true, false, false })] - public void ReadAsBoolean(int value, bool[] bits) + public void ReadAsBoolean(byte value, bool[] bits) { int bitCount = bits.Length; - byte[] buffer = new byte[4]; - BinaryPrimitives.WriteInt32BigEndian(buffer, value << (32 - bitCount)); + byte[] buffer = new byte[8]; + buffer[0] = value; Av1BitStreamReader reader = new(buffer); bool[] actual = new bool[bitCount]; for (int i = 0; i < bitCount; i++) @@ -34,7 +34,7 @@ public class Av1BitsStreamTests [InlineData(4050, 16)] public void ReadAsLiteral(uint expected, int bitCount) { - byte[] buffer = new byte[4]; + byte[] buffer = new byte[8]; BinaryPrimitives.WriteUInt32BigEndian(buffer, expected << (32 - bitCount)); Av1BitStreamReader reader = new(buffer); uint actual = reader.ReadLiteral(bitCount); @@ -46,7 +46,7 @@ public class Av1BitsStreamTests [InlineData(new bool[] { false, true, false, true })] public void WriteAsBoolean(bool[] booleans) { - using MemoryStream stream = new(4); + using MemoryStream stream = new(8); Av1BitStreamWriter writer = new(stream); for (int i = 0; i < booleans.Length; i++) { @@ -73,7 +73,7 @@ public class Av1BitsStreamTests [InlineData(4050, 16)] public void WriteAsLiteral(uint value, int bitCount) { - using MemoryStream stream = new(4); + using MemoryStream stream = new(8); Av1BitStreamWriter writer = new(stream); writer.WriteLiteral(value, bitCount); writer.Flush(); From 894bbf237bc068f02cd204a3cccf3c10812ffa00 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Wed, 17 Apr 2024 23:33:05 +0200 Subject: [PATCH 09/12] Update infra --- .gitattributes | 3 +++ ImageSharp.sln | 17 +++++++++++++++-- shared-infrastructure | 2 +- tests/ImageSharp.Tests/TestImages.cs | 2 ++ tests/Images/Input/Heif/jpeg444_xnconvert.avif | 3 +++ 5 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tests/Images/Input/Heif/jpeg444_xnconvert.avif diff --git a/.gitattributes b/.gitattributes index b5f742ab47..1ab893e0a8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -136,3 +136,6 @@ *.ico filter=lfs diff=lfs merge=lfs -text *.cur filter=lfs diff=lfs merge=lfs -text *.ani filter=lfs diff=lfs merge=lfs -text +*.heic filter=lfs diff=lfs merge=lfs -text +*.hif filter=lfs diff=lfs merge=lfs -text +*.avif filter=lfs diff=lfs merge=lfs -text diff --git a/ImageSharp.sln b/ImageSharp.sln index 162de84168..249c71d6a6 100644 --- a/ImageSharp.sln +++ b/ImageSharp.sln @@ -37,8 +37,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{815C0625-CD3 ProjectSection(SolutionItems) = preProject src\Directory.Build.props = src\Directory.Build.props src\Directory.Build.targets = src\Directory.Build.targets - src\README.md = src\README.md src\ImageSharp.ruleset = src\ImageSharp.ruleset + src\README.md = src\README.md EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ImageSharp", "src\ImageSharp\ImageSharp.csproj", "{2AA31A1F-142C-43F4-8687-09ABCA4B3A26}" @@ -215,6 +215,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "issues", "issues", "{5C9B68 ProjectSection(SolutionItems) = preProject tests\Images\Input\Jpg\issues\issue-1076-invalid-subsampling.jpg = tests\Images\Input\Jpg\issues\issue-1076-invalid-subsampling.jpg tests\Images\Input\Jpg\issues\issue-1221-identify-multi-frame.jpg = tests\Images\Input\Jpg\issues\issue-1221-identify-multi-frame.jpg + tests\Images\Input\Jpg\issues\issue-2067-comment.jpg = tests\Images\Input\Jpg\issues\issue-2067-comment.jpg tests\Images\Input\Jpg\issues\issue1006-incorrect-resize.jpg = tests\Images\Input\Jpg\issues\issue1006-incorrect-resize.jpg tests\Images\Input\Jpg\issues\issue1049-exif-resize.jpg = tests\Images\Input\Jpg\issues\issue1049-exif-resize.jpg tests\Images\Input\Jpg\issues\Issue159-MissingFF00-Progressive-Bedroom.jpg = tests\Images\Input\Jpg\issues\Issue159-MissingFF00-Progressive-Bedroom.jpg @@ -238,7 +239,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "issues", "issues", "{5C9B68 tests\Images\Input\Jpg\issues\issue750-exif-tranform.jpg = tests\Images\Input\Jpg\issues\issue750-exif-tranform.jpg tests\Images\Input\Jpg\issues\Issue845-Incorrect-Quality99.jpg = tests\Images\Input\Jpg\issues\Issue845-Incorrect-Quality99.jpg tests\Images\Input\Jpg\issues\issue855-incorrect-colorspace.jpg = tests\Images\Input\Jpg\issues\issue855-incorrect-colorspace.jpg - tests\Images\Input\Jpg\issues\issue-2067-comment.jpg = tests\Images\Input\Jpg\issues\issue-2067-comment.jpg EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "fuzz", "fuzz", "{516A3532-6AC2-417B-AD79-9BD5D0D378A0}" @@ -661,6 +661,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Qoi", "Qoi", "{E801B508-493 tests\Images\Input\Qoi\wikipedia_008.qoi = tests\Images\Input\Qoi\wikipedia_008.qoi EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Heif", "Heif", "{BA5D603A-C84C-43E5-B300-8BB886B02936}" + ProjectSection(SolutionItems) = preProject + tests\Images\Input\Heif\dwsample-heic-640.heic = tests\Images\Input\Heif\dwsample-heic-640.heic + tests\Images\Input\Heif\image1.heic = tests\Images\Input\Heif\image1.heic + tests\Images\Input\Heif\image2.heic = tests\Images\Input\Heif\image2.heic + tests\Images\Input\Heif\image3.heic = tests\Images\Input\Heif\image3.heic + tests\Images\Input\Heif\image4.heic = tests\Images\Input\Heif\image4.heic + tests\Images\Input\Heif\IMG-20230508-0053.hif = tests\Images\Input\Heif\IMG-20230508-0053.hif + tests\Images\Input\Heif\Irvine_CA.avif = tests\Images\Input\Heif\Irvine_CA.avif + tests\Images\Input\Heif\jpeg444_xnconvert.avif = tests\Images\Input\Heif\jpeg444_xnconvert.avif + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -714,6 +726,7 @@ Global {670DD46C-82E9-499A-B2D2-00A802ED0141} = {E1C42A6F-913B-4A7B-B1A8-2BB62843B254} {5DFC394F-136F-4B76-9BCA-3BA786515EFC} = {9DA226A1-8656-49A8-A58A-A8B5C081AD66} {E801B508-4935-41CD-BA85-CF11BFF55A45} = {9DA226A1-8656-49A8-A58A-A8B5C081AD66} + {BA5D603A-C84C-43E5-B300-8BB886B02936} = {9DA226A1-8656-49A8-A58A-A8B5C081AD66} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {5F8B9D1F-CD8B-4CC5-8216-D531E25BD795} diff --git a/shared-infrastructure b/shared-infrastructure index 1dbfb576c8..922c5b21e5 160000 --- a/shared-infrastructure +++ b/shared-infrastructure @@ -1 +1 @@ -Subproject commit 1dbfb576c83507645265c79e03369b66cdc0379f +Subproject commit 922c5b21e5dfa02d4ef0d95334ab01c87a7a4309 diff --git a/tests/ImageSharp.Tests/TestImages.cs b/tests/ImageSharp.Tests/TestImages.cs index acff77c1a2..8c845eae7d 100644 --- a/tests/ImageSharp.Tests/TestImages.cs +++ b/tests/ImageSharp.Tests/TestImages.cs @@ -1135,5 +1135,7 @@ public static class TestImages // Downloaded from: https://github.com/AOMediaCodec/av1-avif/blob/master/testFiles/Microsoft/Irvine_CA.avif public const string IrvineAvif = "Heif/Irvine_CA.avif"; + + public const string XnConvert = "Heif/jpeg444_xnconvert.avif"; } } diff --git a/tests/Images/Input/Heif/jpeg444_xnconvert.avif b/tests/Images/Input/Heif/jpeg444_xnconvert.avif new file mode 100644 index 0000000000..5080be098d --- /dev/null +++ b/tests/Images/Input/Heif/jpeg444_xnconvert.avif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73c0521c669759df918cd59865537cd9cef3b7502ce3da825d8d24d9997e0e0d +size 1242 From 47dc7746fca3bc7390eb2eaae626cf343eb2204e Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sun, 21 Apr 2024 14:37:55 +0200 Subject: [PATCH 10/12] Obu header decoding --- .../Formats/Heif/Av1/Av1BitStreamReader.cs | 8 +- .../Formats/Heif/Av1/Av1DecoderHandle.cs | 28 ++ ...tDirectionalEnhancementFilterParameters.cs | 4 +- .../Av1/OpenBitstreamUnit/ObuFrameHeader.cs | 2 - .../ObuLoopRestorationParameters.cs | 2 +- .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 321 ++++++++++++------ .../Formats/Heif/Av1/ObuFrameHeaderTests.cs | 32 ++ 7 files changed, 287 insertions(+), 110 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs create mode 100644 tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs index b355a9a5e9..d378f8e034 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs @@ -1,8 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System; -using System.Buffers.Binary; using System.Runtime.InteropServices; namespace SixLabors.ImageSharp.Formats.Heif.Av1; @@ -50,7 +48,7 @@ internal ref struct Av1BitStreamReader public uint ReadLiteral(int bitCount) { - DebugGuard.MustBeBetweenOrEqualTo(bitCount, 1, 32, nameof(bitCount)); + DebugGuard.MustBeBetweenOrEqualTo(bitCount, 0, 32, nameof(bitCount)); uint bits = (this.currentWord << this.bitOffset) >> (WordSize - bitCount); this.bitOffset += bitCount; @@ -77,7 +75,7 @@ internal ref struct Av1BitStreamReader public ulong ReadLittleEndianBytes128(out int length) { // See section 4.10.5 of the AV1-Specification - DebugGuard.IsTrue((this.bitOffset & 0xF7) == 0, "Reading of Little Endian 128 value only allowed on byte alignment"); + DebugGuard.IsTrue((this.bitOffset & 0x07) == 0, $"Reading of Little Endian 128 value only allowed on byte alignment (offset {this.BitPosition})."); ulong value = 0; length = 0; @@ -122,7 +120,7 @@ internal ref struct Av1BitStreamReader return 0; } - int w = (int)(Av1Math.MostSignificantBit(n) + 1); + int w = (int)(Av1Math.FloorLog2(n) + 1); uint m = (uint)((1 << w) - n); uint v = this.ReadLiteral(w - 1); if (v < m) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs b/src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs new file mode 100644 index 0000000000..cdad5a8905 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs @@ -0,0 +1,28 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal class Av1DecoderHandle +{ + public Av1DecoderHandle() + { + this.FrameInfo = new ObuFrameHeader(); + this.SequenceHeader = new ObuSequenceHeader(); + this.TileInfo = new ObuTileInfo(); + } + + public bool SequenceHeaderDone { get; set; } + + public bool ShowExistingFrame { get; set; } + + public bool SeenFrameHeader { get; set; } + + public ObuFrameHeader FrameInfo { get; } + + public ObuSequenceHeader SequenceHeader { get; } + + public ObuTileInfo TileInfo { get; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs index 044e7799e5..ffd561476a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuConstraintDirectionalEnhancementFilterParameters.cs @@ -7,9 +7,9 @@ internal class ObuConstraintDirectionalEnhancementFilterParameters { public int BitCount { get; internal set; } - public int[] YStrength { get; internal set; } = new int[4]; + public int[] YStrength { get; internal set; } = new int[5]; - public int[] UVStrength { get; internal set; } = new int[4]; + public int[] UVStrength { get; internal set; } = new int[5]; public int Damping { get; internal set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs index 315f0f18f2..7ebca0c235 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -77,8 +77,6 @@ internal class ObuFrameHeader internal bool DisableCdfUpdate { get; set; } - internal bool ForeceIntegerMotionVector { get; set; } - internal uint CurrentFrameId { get; set; } internal uint[] ReferenceFrameIndex { get; set; } = new uint[ObuConstants.ReferenceFrameCount]; diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs index 1c04072e3e..4a08041ec2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuLoopRestorationParameters.cs @@ -5,5 +5,5 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuLoopRestorationParameters { - public ObuRestorationType FrameRestorationType { get; internal set; } + public ObuRestorationType FrameRestorationType { get; internal set; } = ObuRestorationType.RestoreNone; } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index b9a6425779..485fee58dc 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -7,21 +7,117 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuReader { - private static readonly int[] WienerTapsMid = new[] { 3, -7, 15 }; - private static readonly int[] SgrprojXqdMid = new[] { -32, 31 }; + /// + /// Decode all OBU's in a frame. + /// + public static void Read(ref Av1BitStreamReader reader, int dataSize, Av1DecoderHandle decoderHandle, bool isAnnexB) + { + bool frameDecodingFinished = false; + while (!frameDecodingFinished) + { + int lengthSize = 0; + int payloadSize = 0; + if (isAnnexB) + { + ReadObuSize(ref reader, out payloadSize, out lengthSize); + } + + ObuHeader header = ReadObuHeaderSize(ref reader, out lengthSize); + if (isAnnexB) + { + header.PayloadSize -= header.Size; + dataSize -= lengthSize; + lengthSize = 0; + } + + payloadSize = header.PayloadSize; + dataSize -= header.Size + lengthSize; + if (isAnnexB && dataSize < payloadSize) + { + throw new InvalidImageContentException("Corrupt frame"); + } + + switch (header.Type) + { + case ObuType.SequenceHeader: + ReadSequenceHeader(ref reader, decoderHandle.SequenceHeader); + if (decoderHandle.SequenceHeader.ColorConfig.BitDepth == 12) + { + // TODO: Initialize 12 bit predictors + } + + decoderHandle.SequenceHeaderDone = true; + break; + case ObuType.FrameHeader: + case ObuType.RedundantFrameHeader: + case ObuType.Frame: + if (header.Type != ObuType.Frame) + { + decoderHandle.ShowExistingFrame = false; + } + else if (header.Type != ObuType.FrameHeader) + { + Guard.IsFalse(decoderHandle.SeenFrameHeader, nameof(Av1DecoderHandle.SeenFrameHeader), "Frame header expected"); + } + else + { + Guard.IsTrue(decoderHandle.SeenFrameHeader, nameof(Av1DecoderHandle.SeenFrameHeader), "Already decoded a frame header"); + } + + if (!decoderHandle.SeenFrameHeader) + { + decoderHandle.SeenFrameHeader = true; + ReadFrameHeader(ref reader, decoderHandle.SequenceHeader, decoderHandle.FrameInfo, header, header.Type != ObuType.Frame); + } + + if (header.Type != ObuType.Frame) + { + break; // For OBU_TILE_GROUP comes under OBU_FRAME + } + + goto TILE_GROUP; + case ObuType.TileGroup: + TILE_GROUP: + if (!decoderHandle.SeenFrameHeader) + { + throw new InvalidImageContentException("Corrupt frame"); + } + + ReadTileGroup(ref reader, decoderHandle.SequenceHeader, decoderHandle.FrameInfo, decoderHandle.TileInfo, header, out frameDecodingFinished); + if (frameDecodingFinished) + { + decoderHandle.SeenFrameHeader = false; + } - private static ObuHeader ReadObuHeader(Av1BitStreamReader reader) + break; + case ObuType.TemporalDelimiter: + default: + // Ignore unknown OBU types. + // throw new InvalidImageContentException($"Unknown OBU header found: {header.Type.ToString()}"); + break; + } + + dataSize -= payloadSize; + if (dataSize <= 0) + { + frameDecodingFinished = true; + } + } + } + + private static ObuHeader ReadObuHeader(ref Av1BitStreamReader reader) { ObuHeader header = new(); - if (!reader.ReadBoolean()) + if (reader.ReadBoolean()) { throw new ImageFormatException("Forbidden bit in header should be unset."); } + header.Size = 1; header.Type = (ObuType)reader.ReadLiteral(4); header.HasExtension = reader.ReadBoolean(); header.HasSize = reader.ReadBoolean(); - if (!reader.ReadBoolean()) + if (reader.ReadBoolean()) { throw new ImageFormatException("Reserved bit in header should be unset."); } @@ -45,9 +141,9 @@ internal class ObuReader return header; } - private static void ReadObuSize(Av1BitStreamReader reader, out int obuSize) + private static void ReadObuSize(ref Av1BitStreamReader reader, out int obuSize, out int lengthSize) { - ulong rawSize = reader.ReadLittleEndianBytes128(out _); + ulong rawSize = reader.ReadLittleEndianBytes128(out lengthSize); if (rawSize > uint.MaxValue) { throw new ImageFormatException("OBU block too large."); @@ -59,12 +155,13 @@ internal class ObuReader /// /// Read OBU header and size. /// - private static ObuHeader ReadObuHeaderSize(Av1BitStreamReader reader) + private static ObuHeader ReadObuHeaderSize(ref Av1BitStreamReader reader, out int lengthSize) { - ObuHeader header = ReadObuHeader(reader); + ObuHeader header = ReadObuHeader(ref reader); + lengthSize = 0; if (header.HasSize) { - ReadObuSize(reader, out int payloadSize); + ReadObuSize(ref reader, out int payloadSize, out lengthSize); header.PayloadSize = payloadSize; } @@ -75,17 +172,17 @@ internal class ObuReader /// Check that the trailing bits start with a 1 and end with 0s. /// /// Consumes a byte, if already byte aligned before the check. - private static void ReadTrailingBits(Av1BitStreamReader reader) + private static void ReadTrailingBits(ref Av1BitStreamReader reader) { int bitsBeforeAlignment = 8 - (reader.BitPosition & 0x7); uint trailing = reader.ReadLiteral(bitsBeforeAlignment); - if (trailing != (1 << (bitsBeforeAlignment - 1))) + if (trailing != (1U << (bitsBeforeAlignment - 1))) { throw new ImageFormatException("Trailing bits not properly formatted."); } } - private static void AlignToByteBoundary(Av1BitStreamReader reader) + private static void AlignToByteBoundary(ref Av1BitStreamReader reader) { while ((reader.BitPosition & 0x7) > 0) { @@ -111,9 +208,8 @@ internal class ObuReader _ => false, }; - private static ObuSequenceHeader ReadSequenceHeader(Av1BitStreamReader reader) + private static void ReadSequenceHeader(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader) { - ObuSequenceHeader sequenceHeader = new(); sequenceHeader.SequenceProfile = (ObuSequenceProfile)reader.ReadLiteral(3); if (sequenceHeader.SequenceProfile > ObuConstants.MaxSequenceProfile) { @@ -130,16 +226,19 @@ internal class ObuReader sequenceHeader.TimingInfo = null; sequenceHeader.DecoderModelInfoPresentFlag = false; sequenceHeader.InitialDisplayDelayPresentFlag = false; - sequenceHeader.OperatingPoint[0].OperatorIndex = 0; - sequenceHeader.OperatingPoint[0].SequenceLevelIndex = (int)reader.ReadLiteral(ObuConstants.LevelBits); + sequenceHeader.OperatingPoint = new ObuOperatingPoint[1]; + ObuOperatingPoint operatingPoint = new(); + sequenceHeader.OperatingPoint[0] = operatingPoint; + operatingPoint.OperatorIndex = 0; + operatingPoint.SequenceLevelIndex = (int)reader.ReadLiteral(ObuConstants.LevelBits); if (!IsValidSequenceLevel(sequenceHeader.OperatingPoint[0].SequenceLevelIndex)) { - throw new ImageFormatException("Unknown sequnce level."); + throw new ImageFormatException("Invalid sequence level."); } - sequenceHeader.OperatingPoint[0].SequenceTier = 0; - sequenceHeader.OperatingPoint[0].IsDecoderModelPresent = false; - sequenceHeader.OperatingPoint[0].IsInitialDisplayDelayPresent = false; + operatingPoint.SequenceTier = 0; + operatingPoint.IsDecoderModelPresent = false; + operatingPoint.IsInitialDisplayDelayPresent = false; // Video related flags removed @@ -172,18 +271,17 @@ internal class ObuReader sequenceHeader.EnableSuperResolution = reader.ReadBoolean(); sequenceHeader.CdefLevel = (int)reader.ReadLiteral(1); sequenceHeader.EnableRestoration = reader.ReadBoolean(); - sequenceHeader.ColorConfig = ReadColorConfig(reader, sequenceHeader); + sequenceHeader.ColorConfig = ReadColorConfig(ref reader, sequenceHeader); sequenceHeader.AreFilmGrainingParametersPresent = reader.ReadBoolean(); - ReadTrailingBits(reader); - return sequenceHeader; + ReadTrailingBits(ref reader); } - private static ObuColorConfig ReadColorConfig(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader) + private static ObuColorConfig ReadColorConfig(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader) { ObuColorConfig colorConfig = new(); - ReadBitDepth(reader, colorConfig, sequenceHeader); + ReadBitDepth(ref reader, colorConfig, sequenceHeader); colorConfig.Monochrome = false; - if (sequenceHeader.SequenceProfile == ObuSequenceProfile.High) + if (sequenceHeader.SequenceProfile != ObuSequenceProfile.High) { colorConfig.Monochrome = reader.ReadBoolean(); } @@ -224,21 +322,33 @@ internal class ObuReader else { colorConfig.ColorRange = reader.ReadBoolean(); - if (sequenceHeader.SequenceProfile != ObuSequenceProfile.Main) + switch (sequenceHeader.SequenceProfile) { - if (colorConfig.BitDepth == 12) - { - colorConfig.SubSamplingX = reader.ReadBoolean(); - if (colorConfig.SubSamplingX) - { - colorConfig.SubSamplingY = reader.ReadBoolean(); - } - } - else - { + case ObuSequenceProfile.Main: colorConfig.SubSamplingX = true; + colorConfig.SubSamplingY = true; + break; + case ObuSequenceProfile.High: + colorConfig.SubSamplingX = false; colorConfig.SubSamplingY = false; - } + break; + case ObuSequenceProfile.Professional: + default: + if (colorConfig.BitDepth == 12) + { + colorConfig.SubSamplingX = reader.ReadBoolean(); + if (colorConfig.SubSamplingX) + { + colorConfig.SubSamplingY = reader.ReadBoolean(); + } + } + else + { + colorConfig.SubSamplingX = true; + colorConfig.SubSamplingY = false; + } + + break; } if (colorConfig.SubSamplingX && colorConfig.SubSamplingY) @@ -251,7 +361,7 @@ internal class ObuReader return colorConfig; } - private static void ReadBitDepth(Av1BitStreamReader reader, ObuColorConfig colorConfig, ObuSequenceHeader sequenceHeader) + private static void ReadBitDepth(ref Av1BitStreamReader reader, ObuColorConfig colorConfig, ObuSequenceHeader sequenceHeader) { bool hasHighBitDepth = reader.ReadBoolean(); if (sequenceHeader.SequenceProfile == ObuSequenceProfile.Professional && hasHighBitDepth) @@ -264,7 +374,7 @@ internal class ObuReader } } - private static void ReadSuperResolutionParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + private static void ReadSuperResolutionParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) { bool useSuperResolution = false; if (sequenceHeader.EnableSuperResolution) @@ -283,8 +393,9 @@ internal class ObuReader frameInfo.FrameSize.SuperResolutionUpscaledWidth = frameInfo.FrameSize.FrameWidth; frameInfo.FrameSize.FrameWidth = - (frameInfo.FrameSize.SuperResolutionUpscaledWidth * ObuConstants.ScaleNumerator) + - (frameInfo.FrameSize.SuperResolutionDenominator / 2); + ((frameInfo.FrameSize.SuperResolutionUpscaledWidth * ObuConstants.ScaleNumerator) + + (frameInfo.FrameSize.SuperResolutionDenominator / 2)) / + frameInfo.FrameSize.SuperResolutionDenominator; if (frameInfo.FrameSize.SuperResolutionDenominator != ObuConstants.ScaleNumerator) { @@ -293,7 +404,7 @@ internal class ObuReader } } - private static void ReadRenderSize(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + private static void ReadRenderSize(ref Av1BitStreamReader reader, ObuFrameHeader frameInfo) { bool renderSizeAndFrameSizeDifferent = reader.ReadBoolean(); if (renderSizeAndFrameSizeDifferent) @@ -308,7 +419,7 @@ internal class ObuReader } } - private static void ReadFrameSizeWithReferences(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool frameSizeOverrideFlag) + private static void ReadFrameSizeWithReferences(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool frameSizeOverrideFlag) { bool foundReference = false; for (int i = 0; i < ObuConstants.ReferencesPerFrame; i++) @@ -323,17 +434,17 @@ internal class ObuReader if (!foundReference) { - ReadFrameSize(reader, sequenceHeader, frameInfo, frameSizeOverrideFlag); - ReadRenderSize(reader, frameInfo); + ReadFrameSize(ref reader, sequenceHeader, frameInfo, frameSizeOverrideFlag); + ReadRenderSize(ref reader, frameInfo); } else { - ReadSuperResolutionParameters(reader, sequenceHeader, frameInfo); + ReadSuperResolutionParameters(ref reader, sequenceHeader, frameInfo); ComputeImageSize(sequenceHeader, frameInfo); } } - private static void ReadFrameSize(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool frameSizeOverrideFlag) + private static void ReadFrameSize(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool frameSizeOverrideFlag) { if (frameSizeOverrideFlag) { @@ -346,11 +457,11 @@ internal class ObuReader frameInfo.FrameSize.FrameHeight = sequenceHeader.MaxFrameHeight; } - ReadSuperResolutionParameters(reader, sequenceHeader, frameInfo); + ReadSuperResolutionParameters(ref reader, sequenceHeader, frameInfo); ComputeImageSize(sequenceHeader, frameInfo); } - private static ObuTileInfo ReadTileInfo(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + private static ObuTileInfo ReadTileInfo(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) { ObuTileInfo tileInfo = new(); int superBlockColumnCount; @@ -518,7 +629,7 @@ internal class ObuReader return tileInfo; } - private static void ReadUncompressedFrameHeader(Av1BitStreamReader reader, ObuHeader header, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private static void ReadUncompressedFrameHeader(ref Av1BitStreamReader reader, ObuHeader header, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) { int idLength = 0; uint previousFrameId = 0; @@ -562,7 +673,7 @@ internal class ObuReader { if (sequenceHeader.SequenceForceIntegerMotionVector == 1) { - frameInfo.ForeceIntegerMotionVector = reader.ReadBoolean(); + frameInfo.ForceIntegerMotionVector = reader.ReadBoolean(); } else { @@ -571,12 +682,12 @@ internal class ObuReader } else { - frameInfo.ForeceIntegerMotionVector = false; + frameInfo.ForceIntegerMotionVector = false; } if (isIntraFrame) { - frameInfo.ForeceIntegerMotionVector = true; + frameInfo.ForceIntegerMotionVector = true; } bool havePreviousFrameId = !(frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame); @@ -635,7 +746,12 @@ internal class ObuReader frameSizeOverrideFlag = reader.ReadBoolean(); } - frameInfo.OrderHint = reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); + frameInfo.OrderHint = 0; + if (sequenceHeader.OrderHintInfo.OrderHintBits > 0) + { + frameInfo.OrderHint = reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); + } + if (isIntraFrame || frameInfo.ErrorResilientMode) { frameInfo.PrimaryReferenceFrame = ObuConstants.PrimaryReferenceFrameNone; @@ -651,7 +767,7 @@ internal class ObuReader frameInfo.AllowIntraBlockCopy = false; if (frameInfo.FrameType == ObuFrameType.SwitchFrame || (frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame)) { - frameInfo.RefreshFrameFlags = 0xffU; + frameInfo.RefreshFrameFlags = 0xFFU; } else { @@ -680,8 +796,8 @@ internal class ObuReader if (isIntraFrame) { - ReadFrameSize(reader, sequenceHeader, frameInfo, frameSizeOverrideFlag); - ReadRenderSize(reader, frameInfo); + ReadFrameSize(ref reader, sequenceHeader, frameInfo, frameSizeOverrideFlag); + ReadRenderSize(ref reader, frameInfo); if (frameInfo.AllowScreenContentTools && frameInfo.FrameSize.RenderWidth != 0) { if (frameInfo.FrameSize.FrameWidth == frameInfo.FrameSize.SuperResolutionUpscaledWidth) @@ -706,15 +822,15 @@ internal class ObuReader if (frameInfo.PrimaryReferenceFrame == ObuConstants.PrimaryReferenceFrameNone) { - // SetupPastIndependence(); + SetupPastIndependence(frameInfo); } // GenerateNextReferenceFrameMap(sequenceHeader, frameInfo); - frameInfo.TilesInfo = ReadTileInfo(reader, sequenceHeader, frameInfo); - ReadQuantizationParameters(reader, frameInfo.QuantizationParameters, sequenceHeader.ColorConfig, planesCount); - ReadSegmentationParameters(reader, sequenceHeader, frameInfo, planesCount); - ReadFrameDeltaQParameters(reader, frameInfo); - ReadFrameDeltaLoopFilterParameters(reader, frameInfo); + frameInfo.TilesInfo = ReadTileInfo(ref reader, sequenceHeader, frameInfo); + ReadQuantizationParameters(ref reader, frameInfo.QuantizationParameters, sequenceHeader.ColorConfig, planesCount); + ReadSegmentationParameters(ref reader, sequenceHeader, frameInfo, planesCount); + ReadFrameDeltaQParameters(ref reader, frameInfo); + ReadFrameDeltaLoopFilterParameters(ref reader, frameInfo); // SetupSegmentationDequantization(); Av1MainParseContext mainParseContext = new(); @@ -758,21 +874,26 @@ internal class ObuReader } frameInfo.AllLossless = frameInfo.CodedLossless && frameInfo.FrameSize.FrameWidth == frameInfo.FrameSize.SuperResolutionUpscaledWidth; - ReadLoopFilterParameters(reader, sequenceHeader, frameInfo, planesCount); - ReadCdefParameters(reader, sequenceHeader, frameInfo, planesCount); - ReadLoopRestorationParameters(reader, sequenceHeader, frameInfo, planesCount); - ReadTransformMode(reader, frameInfo); + ReadLoopFilterParameters(ref reader, sequenceHeader, frameInfo, planesCount); + ReadCdefParameters(ref reader, sequenceHeader, frameInfo, planesCount); + ReadLoopRestorationParameters(ref reader, sequenceHeader, frameInfo, planesCount); + ReadTransformMode(ref reader, frameInfo); - frameInfo.ReferenceMode = ReadFrameReferenceMode(reader, isIntraFrame); - ReadSkipModeParameters(reader, sequenceHeader, frameInfo, isIntraFrame, frameInfo.ReferenceMode); + frameInfo.ReferenceMode = ReadFrameReferenceMode(ref reader, isIntraFrame); + ReadSkipModeParameters(ref reader, sequenceHeader, frameInfo, isIntraFrame, frameInfo.ReferenceMode); if (isIntraFrame || frameInfo.ErrorResilientMode || !sequenceHeader.EnableWarpedMotion) { frameInfo.AllowWarpedMotion = false; } frameInfo.ReducedTransformSet = reader.ReadBoolean(); - ReadGlobalMotionParameters(reader, sequenceHeader, frameInfo, isIntraFrame); - frameInfo.FilmGrainParameters = ReadFilmGrainFilterParameters(reader, sequenceHeader, frameInfo); + ReadGlobalMotionParameters(ref reader, sequenceHeader, frameInfo, isIntraFrame); + frameInfo.FilmGrainParameters = ReadFilmGrainFilterParameters(ref reader, sequenceHeader, frameInfo); + } + + private static void SetupPastIndependence(ObuFrameHeader frameInfo) + { + // TODO: Initialize the loop filter parameters. } private static bool IsSegmentationFeatureActive(ObuSegmentationParameters segmentationParameters, int segmentId, ObuSegmentationLevelFeature feature) @@ -792,24 +913,24 @@ internal class ObuReader } } - private static void ReadFrameHeader(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuHeader header, bool trailingBit) + private static void ReadFrameHeader(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuHeader header, bool trailingBit) { int planeCount = sequenceHeader.ColorConfig.Monochrome ? 1 : 3; int startBitPosition = reader.BitPosition; - ReadUncompressedFrameHeader(reader, header, sequenceHeader, frameInfo, planeCount); + ReadUncompressedFrameHeader(ref reader, header, sequenceHeader, frameInfo, planeCount); if (trailingBit) { - ReadTrailingBits(reader); + ReadTrailingBits(ref reader); } - AlignToByteBoundary(reader); + AlignToByteBoundary(ref reader); int endPosition = reader.BitPosition; int headerBytes = (endPosition - startBitPosition) / 8; header.PayloadSize -= headerBytes; } - private static void ReadTileGroup(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileInfo tileInfo, ObuHeader header, out bool isLastTileGroup) + private static void ReadTileGroup(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileInfo tileInfo, ObuHeader header, out bool isLastTileGroup) { int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; int startBitPosition = reader.BitPosition; @@ -834,7 +955,7 @@ internal class ObuReader } isLastTileGroup = (tileGroupEnd + 1) == tileCount; - AlignToByteBoundary(reader); + AlignToByteBoundary(ref reader); int endBitPosition = reader.BitPosition; int headerBytes = (endBitPosition - startBitPosition) / 8; header.PayloadSize -= headerBytes; @@ -874,7 +995,7 @@ internal class ObuReader // FinishDecodeTiles(sequenceHeader, frameInfo, doCdef, doLoopRestoration); } - private static int ReadDeltaQ(Av1BitStreamReader reader) + private static int ReadDeltaQ(ref Av1BitStreamReader reader) { int deltaQ = 0; if (reader.ReadBoolean()) @@ -885,7 +1006,7 @@ internal class ObuReader return deltaQ; } - private static void ReadFrameDeltaQParameters(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + private static void ReadFrameDeltaQParameters(ref Av1BitStreamReader reader, ObuFrameHeader frameInfo) { frameInfo.DeltaQParameters.Resolution = 0; frameInfo.DeltaQParameters.IsPresent = false; @@ -900,7 +1021,7 @@ internal class ObuReader } } - private static void ReadFrameDeltaLoopFilterParameters(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + private static void ReadFrameDeltaLoopFilterParameters(ref Av1BitStreamReader reader, ObuFrameHeader frameInfo) { frameInfo.DeltaLoopFilterParameters.IsPresent = false; frameInfo.DeltaLoopFilterParameters.Resolution = 0; @@ -920,20 +1041,20 @@ internal class ObuReader } } - private static void ReadQuantizationParameters(Av1BitStreamReader reader, ObuQuantizationParameters quantParams, ObuColorConfig colorInfo, int planesCount) + private static void ReadQuantizationParameters(ref Av1BitStreamReader reader, ObuQuantizationParameters quantParams, ObuColorConfig colorInfo, int planesCount) { quantParams.BaseQIndex = (int)reader.ReadLiteral(8); - quantParams.DeltaQDc[(int)Av1Plane.Y] = ReadDeltaQ(reader); + quantParams.DeltaQDc[(int)Av1Plane.Y] = ReadDeltaQ(ref reader); quantParams.DeltaQAc[(int)Av1Plane.Y] = 0; if (planesCount > 1) { bool areUvDeltaDifferent = false; - quantParams.DeltaQDc[(int)Av1Plane.U] = ReadDeltaQ(reader); - quantParams.DeltaQAc[(int)Av1Plane.U] = ReadDeltaQ(reader); + quantParams.DeltaQDc[(int)Av1Plane.U] = ReadDeltaQ(ref reader); + quantParams.DeltaQAc[(int)Av1Plane.U] = ReadDeltaQ(ref reader); if (areUvDeltaDifferent) { - quantParams.DeltaQDc[(int)Av1Plane.V] = ReadDeltaQ(reader); - quantParams.DeltaQAc[(int)Av1Plane.V] = ReadDeltaQ(reader); + quantParams.DeltaQDc[(int)Av1Plane.V] = ReadDeltaQ(ref reader); + quantParams.DeltaQAc[(int)Av1Plane.V] = ReadDeltaQ(ref reader); } else { @@ -971,7 +1092,7 @@ internal class ObuReader } } - private static void ReadSegmentationParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private static void ReadSegmentationParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) { frameInfo.SegmentationParameters.SegmentationEnabled = reader.ReadBoolean(); if (!frameInfo.SegmentationParameters.SegmentationEnabled) @@ -983,7 +1104,7 @@ internal class ObuReader // TODO: Parse more stuff. } - private static void ReadLoopFilterParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private static void ReadLoopFilterParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) { if (frameInfo.CodedLossless || frameInfo.AllowIntraBlockCopy) { @@ -995,7 +1116,7 @@ internal class ObuReader // TODO: Parse more stuff. } - private static void ReadTransformMode(Av1BitStreamReader reader, ObuFrameHeader frameInfo) + private static void ReadTransformMode(ref Av1BitStreamReader reader, ObuFrameHeader frameInfo) { if (frameInfo.CodedLossless) { @@ -1014,21 +1135,21 @@ internal class ObuReader } } - private static void ReadLoopRestorationParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private static void ReadLoopRestorationParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) { _ = planesCount; if (frameInfo.CodedLossless || frameInfo.AllowIntraBlockCopy || !sequenceHeader.EnableRestoration) { - frameInfo.LoopRestorationParameters[0].FrameRestorationType = ObuRestorationType.RestoreNone; - frameInfo.LoopRestorationParameters[1].FrameRestorationType = ObuRestorationType.RestoreNone; - frameInfo.LoopRestorationParameters[2].FrameRestorationType = ObuRestorationType.RestoreNone; + frameInfo.LoopRestorationParameters[0] = new ObuLoopRestorationParameters(); + frameInfo.LoopRestorationParameters[1] = new ObuLoopRestorationParameters(); + frameInfo.LoopRestorationParameters[2] = new ObuLoopRestorationParameters(); return; } // TODO: Parse more stuff. } - private static void ReadCdefParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private static void ReadCdefParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) { _ = planesCount; if (frameInfo.CodedLossless || frameInfo.AllowIntraBlockCopy || sequenceHeader.CdefLevel == 0) @@ -1045,7 +1166,7 @@ internal class ObuReader // TODO: Parse more stuff. } - private static void ReadGlobalMotionParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool isIntraFrame) + private static void ReadGlobalMotionParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool isIntraFrame) { _ = reader; _ = sequenceHeader; @@ -1058,7 +1179,7 @@ internal class ObuReader // TODO: Parse more stuff. } - private static ObuReferenceMode ReadFrameReferenceMode(Av1BitStreamReader reader, bool isIntraFrame) + private static ObuReferenceMode ReadFrameReferenceMode(ref Av1BitStreamReader reader, bool isIntraFrame) { if (isIntraFrame) { @@ -1068,7 +1189,7 @@ internal class ObuReader return (ObuReferenceMode)reader.ReadLiteral(1); } - private static void ReadSkipModeParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool isIntraFrame, ObuReferenceMode referenceSelect) + private static void ReadSkipModeParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool isIntraFrame, ObuReferenceMode referenceSelect) { if (isIntraFrame || referenceSelect == ObuReferenceMode.ReferenceModeSelect || !sequenceHeader.OrderHintInfo.EnableOrderHint) { @@ -1089,7 +1210,7 @@ internal class ObuReader } } - private static ObuFilmGrainParameters ReadFilmGrainFilterParameters(Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + private static ObuFilmGrainParameters ReadFilmGrainFilterParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) { ObuFilmGrainParameters grainParams = new(); if (!sequenceHeader.AreFilmGrainingParametersPresent || (!frameInfo.ShowFrame && !frameInfo.ShowableFrame)) diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs new file mode 100644 index 0000000000..fc02e17c22 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs @@ -0,0 +1,32 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +[Trait("Format", "Avif")] +public class ObuFrameHeaderTests +{ + [Theory] + // [InlineData(TestImages.Heif.IrvineAvif, 0x0102, 0x000D, false)] + // [InlineData(TestImages.Heif.IrvineAvif, 0x0198, 0x6BD1, false)] + [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x0017, false)] + public void ReadFrameHeader(string filename, int fileOffset, int blockSize, bool isAnnexB) + { + // Assign + string filePath = Path.Combine(TestEnvironment.InputImagesDirectoryFullPath, filename); + byte[] content = File.ReadAllBytes(filePath); + Av1BitStreamReader reader = new(content.AsSpan(fileOffset)); + Av1DecoderHandle decoder = new(); + + // Act + ObuReader.Read(ref reader, blockSize, decoder, isAnnexB); + + // Assert + Assert.True(decoder.SequenceHeaderDone); + Assert.False(decoder.SeenFrameHeader); + } + +} From 6aa74f20b972e548a0d3df191e5f9fac0a1655b6 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sun, 21 Apr 2024 14:51:55 +0200 Subject: [PATCH 11/12] Rename to Av1Decoder --- .../Heif/Av1/{Av1DecoderHandle.cs => Av1Decoder.cs} | 10 ++++++++-- .../Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 6 +++--- .../Formats/Heif/Av1/ObuFrameHeaderTests.cs | 8 ++++---- 3 files changed, 15 insertions(+), 9 deletions(-) rename src/ImageSharp/Formats/Heif/Av1/{Av1DecoderHandle.cs => Av1Decoder.cs} (74%) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs similarity index 74% rename from src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs rename to src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index cdad5a8905..7653fea9ce 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1DecoderHandle.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -5,9 +5,9 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; namespace SixLabors.ImageSharp.Formats.Heif.Av1; -internal class Av1DecoderHandle +internal class Av1Decoder { - public Av1DecoderHandle() + public Av1Decoder() { this.FrameInfo = new ObuFrameHeader(); this.SequenceHeader = new ObuSequenceHeader(); @@ -25,4 +25,10 @@ internal class Av1DecoderHandle public ObuSequenceHeader SequenceHeader { get; } public ObuTileInfo TileInfo { get; } + + public void Decode(Span buffer) + { + Av1BitStreamReader reader = new(buffer); + ObuReader.Read(ref reader, buffer.Length, this, false); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index 485fee58dc..656f3c57ea 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -10,7 +10,7 @@ internal class ObuReader /// /// Decode all OBU's in a frame. /// - public static void Read(ref Av1BitStreamReader reader, int dataSize, Av1DecoderHandle decoderHandle, bool isAnnexB) + public static void Read(ref Av1BitStreamReader reader, int dataSize, Av1Decoder decoderHandle, bool isAnnexB) { bool frameDecodingFinished = false; while (!frameDecodingFinished) @@ -57,11 +57,11 @@ internal class ObuReader } else if (header.Type != ObuType.FrameHeader) { - Guard.IsFalse(decoderHandle.SeenFrameHeader, nameof(Av1DecoderHandle.SeenFrameHeader), "Frame header expected"); + Guard.IsFalse(decoderHandle.SeenFrameHeader, nameof(Av1Decoder.SeenFrameHeader), "Frame header expected"); } else { - Guard.IsTrue(decoderHandle.SeenFrameHeader, nameof(Av1DecoderHandle.SeenFrameHeader), "Already decoded a frame header"); + Guard.IsTrue(decoderHandle.SeenFrameHeader, nameof(Av1Decoder.SeenFrameHeader), "Already decoded a frame header"); } if (!decoderHandle.SeenFrameHeader) diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs index fc02e17c22..398f84b853 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs @@ -12,17 +12,17 @@ public class ObuFrameHeaderTests [Theory] // [InlineData(TestImages.Heif.IrvineAvif, 0x0102, 0x000D, false)] // [InlineData(TestImages.Heif.IrvineAvif, 0x0198, 0x6BD1, false)] - [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x0017, false)] + [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC, false)] public void ReadFrameHeader(string filename, int fileOffset, int blockSize, bool isAnnexB) { // Assign string filePath = Path.Combine(TestEnvironment.InputImagesDirectoryFullPath, filename); byte[] content = File.ReadAllBytes(filePath); - Av1BitStreamReader reader = new(content.AsSpan(fileOffset)); - Av1DecoderHandle decoder = new(); + Span span = content.AsSpan(fileOffset, blockSize); + Av1Decoder decoder = new(); // Act - ObuReader.Read(ref reader, blockSize, decoder, isAnnexB); + decoder.Decode(span); // Assert Assert.True(decoder.SequenceHeaderDone); From f2bfd5a25ae9223c365c4287cb315e6249fd53a9 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Tue, 23 Apr 2024 17:52:42 +0200 Subject: [PATCH 12/12] Tile decoding interface --- .../Formats/Heif/Av1/Av1BlockSize.cs | 10 +++++ src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs | 17 +++++++ src/ImageSharp/Formats/Heif/Av1/Av1Math.cs | 21 ++++++++- .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 45 +++++++++++-------- src/ImageSharp/ImageSharp.csproj | 4 -- 5 files changed, 73 insertions(+), 24 deletions(-) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs index 6eca5bc638..f043208e98 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BlockSize.cs @@ -32,3 +32,13 @@ internal enum Av1BlockSize BlockInvalid = 255, BlockLargest = BlockSizeS - 1, } + +internal static class Av1BlockSizeExtensions +{ + private static readonly int[] SizeWide = { 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 1, 4, 2, 8, 4, 16 }; + private static readonly int[] SizeHigh = { 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, 16, 8, 16, 32, 16, 32, 4, 1, 8, 2, 16, 4 }; + + public static int Get4x4WideCount(this Av1BlockSize blockSize) => SizeWide[(int)blockSize]; + + public static int Get4x4HighCount(this Av1BlockSize blockSize) => SizeHigh[(int)blockSize]; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index 7653fea9ce..c8bec52428 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -31,4 +31,21 @@ internal class Av1Decoder Av1BitStreamReader reader = new(buffer); ObuReader.Read(ref reader, buffer.Length, this, false); } + + internal void DecodeTile(ref Av1BitStreamReader reader, int tileNum) + { + // TODO: Implement + } + + internal void DecodeBlock(Av1BlockModeInfo blockMode, int rowIndex, int columnIndex) + { + int block4x4Width = blockMode.BlockSize.Get4x4WideCount(); + int block4x4Height = blockMode.BlockSize.Get4x4HighCount(); + + } + + internal void FinishDecodeTiles(ref Av1BitStreamReader reader, bool doCdef, bool doLoopRestoration) + { + // TODO: Implement + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs index 36929e3c65..ce5dd66269 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Math.cs @@ -5,7 +5,26 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1; internal static class Av1Math { - public static uint MostSignificantBit(uint value) => value >> 31; + public static int MostSignificantBit(uint value) + { + int log = 0; + int i; + + Guard.IsTrue(value != 0, nameof(value), "Must have al least 1 bit set"); + + for (i = 4; i >= 0; --i) + { + int shift = 1 << i; + uint x = value >> shift; + if (x != 0) + { + value = x; + log += shift; + } + } + + return log; + } public static uint Log2(uint n) { diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index 656f3c57ea..6afd89a24c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -10,7 +10,7 @@ internal class ObuReader /// /// Decode all OBU's in a frame. /// - public static void Read(ref Av1BitStreamReader reader, int dataSize, Av1Decoder decoderHandle, bool isAnnexB) + public static void Read(ref Av1BitStreamReader reader, int dataSize, Av1Decoder decoder, bool isAnnexB) { bool frameDecodingFinished = false; while (!frameDecodingFinished) @@ -40,34 +40,34 @@ internal class ObuReader switch (header.Type) { case ObuType.SequenceHeader: - ReadSequenceHeader(ref reader, decoderHandle.SequenceHeader); - if (decoderHandle.SequenceHeader.ColorConfig.BitDepth == 12) + ReadSequenceHeader(ref reader, decoder.SequenceHeader); + if (decoder.SequenceHeader.ColorConfig.BitDepth == 12) { // TODO: Initialize 12 bit predictors } - decoderHandle.SequenceHeaderDone = true; + decoder.SequenceHeaderDone = true; break; case ObuType.FrameHeader: case ObuType.RedundantFrameHeader: case ObuType.Frame: if (header.Type != ObuType.Frame) { - decoderHandle.ShowExistingFrame = false; + decoder.ShowExistingFrame = false; } else if (header.Type != ObuType.FrameHeader) { - Guard.IsFalse(decoderHandle.SeenFrameHeader, nameof(Av1Decoder.SeenFrameHeader), "Frame header expected"); + Guard.IsFalse(decoder.SeenFrameHeader, nameof(Av1Decoder.SeenFrameHeader), "Frame header expected"); } else { - Guard.IsTrue(decoderHandle.SeenFrameHeader, nameof(Av1Decoder.SeenFrameHeader), "Already decoded a frame header"); + Guard.IsTrue(decoder.SeenFrameHeader, nameof(Av1Decoder.SeenFrameHeader), "Already decoded a frame header"); } - if (!decoderHandle.SeenFrameHeader) + if (!decoder.SeenFrameHeader) { - decoderHandle.SeenFrameHeader = true; - ReadFrameHeader(ref reader, decoderHandle.SequenceHeader, decoderHandle.FrameInfo, header, header.Type != ObuType.Frame); + decoder.SeenFrameHeader = true; + ReadFrameHeader(ref reader, decoder, header, header.Type != ObuType.Frame); } if (header.Type != ObuType.Frame) @@ -78,15 +78,15 @@ internal class ObuReader goto TILE_GROUP; case ObuType.TileGroup: TILE_GROUP: - if (!decoderHandle.SeenFrameHeader) + if (!decoder.SeenFrameHeader) { throw new InvalidImageContentException("Corrupt frame"); } - ReadTileGroup(ref reader, decoderHandle.SequenceHeader, decoderHandle.FrameInfo, decoderHandle.TileInfo, header, out frameDecodingFinished); + ReadTileGroup(ref reader, decoder, header, out frameDecodingFinished); if (frameDecodingFinished) { - decoderHandle.SeenFrameHeader = false; + decoder.SeenFrameHeader = false; } break; @@ -629,8 +629,10 @@ internal class ObuReader return tileInfo; } - private static void ReadUncompressedFrameHeader(ref Av1BitStreamReader reader, ObuHeader header, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private static void ReadUncompressedFrameHeader(ref Av1BitStreamReader reader, Av1Decoder decoder, ObuHeader header, int planesCount) { + ObuSequenceHeader sequenceHeader = decoder.SequenceHeader; + ObuFrameHeader frameInfo = decoder.FrameInfo; int idLength = 0; uint previousFrameId = 0; bool isIntraFrame = false; @@ -913,11 +915,13 @@ internal class ObuReader } } - private static void ReadFrameHeader(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuHeader header, bool trailingBit) + private static void ReadFrameHeader(ref Av1BitStreamReader reader, Av1Decoder decoder, ObuHeader header, bool trailingBit) { + ObuSequenceHeader sequenceHeader = decoder.SequenceHeader; + ObuFrameHeader frameInfo = decoder.FrameInfo; int planeCount = sequenceHeader.ColorConfig.Monochrome ? 1 : 3; int startBitPosition = reader.BitPosition; - ReadUncompressedFrameHeader(ref reader, header, sequenceHeader, frameInfo, planeCount); + ReadUncompressedFrameHeader(ref reader, decoder, header, planeCount); if (trailingBit) { ReadTrailingBits(ref reader); @@ -930,8 +934,11 @@ internal class ObuReader header.PayloadSize -= headerBytes; } - private static void ReadTileGroup(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileInfo tileInfo, ObuHeader header, out bool isLastTileGroup) + private static void ReadTileGroup(ref Av1BitStreamReader reader, Av1Decoder decoder, ObuHeader header, out bool isLastTileGroup) { + ObuSequenceHeader sequenceHeader = decoder.SequenceHeader; + ObuFrameHeader frameInfo = decoder.FrameInfo; + ObuTileInfo tileInfo = decoder.TileInfo; int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; int startBitPosition = reader.BitPosition; bool tileStartAndEndPresentFlag = false; @@ -984,7 +991,7 @@ internal class ObuReader } // TODO: Pass more info to the decoder. - // DecodeTile(sequenceHeader, frameInfo, tileInfo, tileNum); + decoder.DecodeTile(ref reader, tileNum); } if (tileGroupEnd != tileCount - 1) @@ -992,7 +999,7 @@ internal class ObuReader return; } - // FinishDecodeTiles(sequenceHeader, frameInfo, doCdef, doLoopRestoration); + decoder.FinishDecodeTiles(ref reader, doCdef, doLoopRestoration); } private static int ReadDeltaQ(ref Av1BitStreamReader reader) diff --git a/src/ImageSharp/ImageSharp.csproj b/src/ImageSharp/ImageSharp.csproj index b76a139019..63bc8131bd 100644 --- a/src/ImageSharp/ImageSharp.csproj +++ b/src/ImageSharp/ImageSharp.csproj @@ -241,9 +241,5 @@ - - - -