From 80140c4cefa9d6e43d41dff6d587afc1a0d9ca13 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sat, 6 Apr 2024 10:18:46 +0200 Subject: [PATCH] Inverse quantization --- .../Formats/Heif/Av1/Av1BlockModeInfo.cs | 2 + .../Heif/Av1/Transform/Av1InverseQuantizer.cs | 62 ++++++++++++++- ...Quantization.cs => Av1InverseTransform.cs} | 22 +++--- .../Heif/Av1/Transform/Av1ScanOrder.cs | 27 +++++++ .../Av1/Transform/Av1ScanOrderConstants.cs | 75 +++++++++++++++++++ .../Transform/Av1TransformSizeExtensions.cs | 16 ++++ .../Formats/Heif/HeifDecoderCore.cs | 1 - 7 files changed, 189 insertions(+), 16 deletions(-) rename src/ImageSharp/Formats/Heif/Av1/Transform/{Av1Quantization.cs => Av1InverseTransform.cs} (95%) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs index a0db9473d4..c288919cec 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BlockModeInfo.cs @@ -12,4 +12,6 @@ internal class Av1BlockModeInfo public Av1PredictionMode PredictionMode { get; } public Av1PartitionType Partition { get; } + + public int SegmentId { get; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs index 98443393a9..b52473dbd9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseQuantizer.cs @@ -8,8 +8,66 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; internal class Av1InverseQuantizer { - public static int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) + public static int InverseQuantize(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, ObuPartitionInfo part, Av1BlockModeInfo mode, int[] level, int[] qCoefficients, Av1TransformMode txType, Av1TransformSize txSize, Av1Plane plane) { - return 0; + Av1ScanOrder scanOrder = Av1ScanOrderConstants.GetScanOrder(txSize, txType); + short[] scanIndices = scanOrder.Scan; + int maxValue = (1 << (7 + sequenceHeader.ColorConfig.BitDepth)) - 1; + int minValue = -(1 << (7 + sequenceHeader.ColorConfig.BitDepth)); + bool usingQuantizationMatrix = frameHeader.QuantizationParameters.IsUsingQMatrix; + bool lossless = frameHeader.LosslessArray[mode.SegmentId]; + short[] dequant = []; // Get from DecModCtx + int qmLevel = (lossless || !usingQuantizationMatrix) ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1 : frameHeader.QuantizationParameters.QMatrix[(int)plane]; + byte[] iqMatrix = []; // txType.Is2dTransform() ? Get from DecModCtx + int shift = txSize.GetScale(); + + int coefficientCount = level[0]; + Span levelSpan = level.AsSpan(1); + int lev = levelSpan[0]; + int qCoefficient; + if (lev != 0) + { + int pos = scanIndices[0]; + qCoefficient = (int)(((long)Math.Abs(lev) * GetDequantizationValue(dequant[0], pos, iqMatrix)) & 0xffffff); + qCoefficient >>= shift; + + if (lev < 0) + { + qCoefficient = -qCoefficient; + } + + qCoefficients[0] = Av1Math.Clamp(qCoefficient, minValue, maxValue); + } + + for (int i = 1; i < coefficientCount; i++) + { + lev = levelSpan[i]; + if (lev != 0) + { + int pos = scanIndices[i]; + qCoefficient = (int)(((long)Math.Abs(lev) * GetDequantizationValue(dequant[1], pos, iqMatrix)) & 0xffffff); + qCoefficient >>= shift; + + if (lev < 0) + { + qCoefficient = -qCoefficient; + } + + qCoefficients[pos] = Av1Math.Clamp(qCoefficient, minValue, maxValue); + } + } + + return coefficientCount; + } + + private static int GetDequantizationValue(short dequant, int coefficientIndex, byte[] iqMatrix) + { + int dqv = dequant; + if (iqMatrix != null) + { + dqv = ((iqMatrix[coefficientIndex] * dqv) + (1 << (Av1ScanOrderConstants.QuantizationMatrixLevelBitCount - 1))) >> Av1ScanOrderConstants.QuantizationMatrixLevelBitCount; + } + + return dqv; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransform.cs similarity index 95% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransform.cs index 205ffaaa89..4856c47194 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Quantization.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransform.cs @@ -3,7 +3,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; -internal static class Av1Quantization +internal static class Av1InverseTransform { public static readonly int[,] AcQLookup = new int[3, 256] { @@ -121,18 +121,14 @@ internal static class Av1Quantization public static int GetQzbinFactor(int q, Av1BitDepth bitDepth) { int quant = GetDcQuantization(q, 0, bitDepth); - switch (bitDepth) - { - case Av1BitDepth.EightBit: - return q == 0 ? 64 : (quant < 148 ? 84 : 80); - case Av1BitDepth.TenBit: - return q == 0 ? 64 : (quant < 592 ? 84 : 80); - case Av1BitDepth.TwelveBit: - return q == 0 ? 64 : (quant < 2368 ? 84 : 80); - default: - DebugGuard.IsTrue(false, "bit_depth should be EIGHT_BIT, TEN_BIT or TWELVE_BIT"); - return -1; - } + + // Bit hack to get to: + // EightBit => 148 + // TenBit => 592 + // TwelveBit => 2368 + int shift = (int)bitDepth << 1; + int threshold = (1 << shift) * 148; + return q == 0 ? 64 : (quant < threshold ? 84 : 80); } public static void InvertQuantization(out int quantization, out int shift, int d) diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs new file mode 100644 index 0000000000..a773b5479b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrder.cs @@ -0,0 +1,27 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +internal readonly struct Av1ScanOrder +{ + public Av1ScanOrder(short[]? scan) + { + this.Scan = scan ?? []; + this.IScan = []; + this.Neighbors = []; + } + + public Av1ScanOrder(short[] scan, short[] iscan, short[] neighbors) + { + this.Scan = scan; + this.IScan = iscan; + this.Neighbors = neighbors; + } + + public short[] Scan { get; } + + public short[] IScan { get; } + + public short[] Neighbors { get; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs new file mode 100644 index 0000000000..a014ad2245 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ScanOrderConstants.cs @@ -0,0 +1,75 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +internal static class Av1ScanOrderConstants +{ + public const int QuantizationMatrixLevelBitCount = 4; + public const int QuantizationMatrixLevelCount = 1 << QuantizationMatrixLevelBitCount; + + private static readonly Av1ScanOrder[][] ScanOrders = + [ + + // Transform size 4x4 + [ + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(DefaultScan4x4), + new Av1ScanOrder(MatrixRowScan4x4), + new Av1ScanOrder(MatrixColumnScan4x4), + new Av1ScanOrder(MatrixRowScan4x4), + new Av1ScanOrder(MatrixColumnScan4x4), + new Av1ScanOrder(MatrixRowScan4x4), + new Av1ScanOrder(MatrixColumnScan4x4), + ], + + // Transform size 8x8 + [ + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(DefaultScan8x8), + new Av1ScanOrder(MatrixRowScan8x8), + new Av1ScanOrder(MatrixColumnScan8x8), + new Av1ScanOrder(MatrixRowScan8x8), + new Av1ScanOrder(MatrixColumnScan8x8), + new Av1ScanOrder(MatrixRowScan8x8), + new Av1ScanOrder(MatrixColumnScan8x8), + ], + ]; + + private static readonly short[] DefaultScan4x4 = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; + private static readonly short[] MatrixColumnScan4x4 = [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]; + private static readonly short[] MatrixRowScan4x4 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + + private static readonly short[] DefaultScan8x8 = [0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, + 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, + 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63]; + + private static readonly short[] MatrixColumnScan8x8 = [0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57, 2, 10, 18, 26, 34, 42, + 50, 58, 3, 11, 19, 27, 35, 43, 51, 59, 4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, + 37, 45, 53, 61, 6, 14, 22, 30, 38, 46, 54, 62, 7, 15, 23, 31, 39, 47, 55, 63]; + + private static readonly short[] MatrixRowScan8x8 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]; + + public static Av1ScanOrder GetScanOrder(Av1TransformSize txSize, Av1TransformMode txMode) + => ScanOrders[(int)txSize][(int)txMode]; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs new file mode 100644 index 0000000000..3db631e0b3 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformSizeExtensions.cs @@ -0,0 +1,16 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Quantization; + +internal static class Av1TransformSizeExtensions +{ + private static readonly int[] Size2d = [ + 16, 64, 256, 1024, 4096, 32, 32, 128, 128, 512, 512, 2048, 2048, 64, 64, 256, 256, 1024, 1024]; + + public static int GetScale(this Av1TransformSize size) + { + int pels = Size2d[(int)size]; + return (pels > 1024) ? 2 : (pels > 256) ? 1 : 0; + } +} diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index c76f7bac3f..7aeeac56b4 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -6,7 +6,6 @@ using System.Buffers.Binary; using System.Text; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -using SixLabors.ImageSharp.Formats.Webp; using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata;