diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcFillPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcFillPredictor.cs deleted file mode 100644 index 663ee5062f..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcFillPredictor.cs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block with the sample-domain midpoint when neither top nor left neighbors are available. -/// -internal readonly struct Av1DcFillPredictor : IAv1Predictor -{ - /// - /// The number of samples written to each destination row. - /// - private readonly uint blockWidth; - - /// - /// The number of destination rows. - /// - private readonly uint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1DcFillPredictor(Size blockSize) - { - this.blockWidth = (uint)blockSize.Width; - this.blockHeight = (uint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1DcFillPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (uint)transformSize.GetWidth(); - this.blockHeight = (uint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block with the 8-bit midpoint value. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The unused top-neighbor buffer required by the common predictor signature. - /// The unused left-neighbor buffer required by the common predictor signature. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1DcFillPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - // With no reference edge, AV1 uses the midpoint of the unsigned 8-bit sample domain as the DC predictor. - const byte expectedDc = 0x80; - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte destinationRef = ref destination[0]; - for (uint r = 0; r < this.blockHeight; r++) - { - Unsafe.InitBlock(ref destinationRef, expectedDc, this.blockWidth); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcIntraPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcIntraPredictor.cs new file mode 100644 index 0000000000..b316bfed4c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcIntraPredictor.cs @@ -0,0 +1,245 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Reconstructs AV1 DC intra-prediction blocks from the available neighboring samples. +/// +internal static class Av1DcIntraPredictor +{ + /// + /// Predicts an 8-bit DC block. + /// + /// Whether the prepared left reference is available. + /// Whether the prepared top reference is available. + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + public static void Predict(bool hasLeft, bool hasAbove, Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height) + { + int count = (hasAbove ? width : 0) + (hasLeft ? height : 0); + int sum = (hasAbove ? Sum(above[..width]) : 0) + (hasLeft ? Sum(left[..height]) : 0); + + // Section 7.11.2.2 defines the unsigned midpoint when no reference is available. Otherwise adding half + // the reference count implements the specified rounded mean before extending it across the whole block. + byte prediction = count == 0 ? (byte)128 : (byte)((sum + (count >> 1)) / count); + for (int row = 0; row < height; row++) + { + destination.Slice(row * destinationStride, width).Fill(prediction); + } + } + + /// + /// Predicts a high-bit-depth DC block. + /// + /// Whether the prepared left reference is available. + /// Whether the prepared top reference is available. + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + /// The reconstructed sample precision. + public static void Predict(bool hasLeft, bool hasAbove, Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, int bitDepth) + { + int count = (hasAbove ? width : 0) + (hasLeft ? height : 0); + int sum = (hasAbove ? Sum(above[..width]) : 0) + (hasLeft ? Sum(left[..height]) : 0); + short prediction = count == 0 ? (short)(1 << (bitDepth - 1)) : (short)((sum + (count >> 1)) / count); + for (int row = 0; row < height; row++) + { + destination.Slice(row * destinationStride, width).Fill(prediction); + } + } + + /// + /// Predicts an 8-bit DC block without hardware intrinsics. + /// + /// Whether the prepared left reference is available. + /// Whether the prepared top reference is available. + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + public static void PredictScalar(bool hasLeft, bool hasAbove, Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height) + { + int count = (hasAbove ? width : 0) + (hasLeft ? height : 0); + int sum = (hasAbove ? SumScalar(above[..width]) : 0) + (hasLeft ? SumScalar(left[..height]) : 0); + byte prediction = count == 0 ? (byte)128 : (byte)((sum + (count >> 1)) / count); + for (int row = 0; row < height; row++) + { + ref byte destinationRow = ref destination[row * destinationStride]; + for (int column = 0; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = prediction; + } + } + } + + /// + /// Predicts a high-bit-depth DC block without hardware intrinsics. + /// + /// Whether the prepared left reference is available. + /// Whether the prepared top reference is available. + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + /// The reconstructed sample precision. + public static void PredictScalar(bool hasLeft, bool hasAbove, Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, int bitDepth) + { + int count = (hasAbove ? width : 0) + (hasLeft ? height : 0); + int sum = (hasAbove ? SumScalar(above[..width]) : 0) + (hasLeft ? SumScalar(left[..height]) : 0); + short prediction = count == 0 ? (short)(1 << (bitDepth - 1)) : (short)((sum + (count >> 1)) / count); + for (int row = 0; row < height; row++) + { + ref short destinationRow = ref destination[row * destinationStride]; + for (int column = 0; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = prediction; + } + } + } + + /// + /// Sums 8-bit neighboring samples using the widest available SIMD width. + /// + /// The samples to sum. + /// The exact sum. + private static int Sum(ReadOnlySpan samples) + { + ref byte samplesBase = ref MemoryMarshal.GetReference(samples); + int sum = 0; + int index = 0; + + if (Vector512.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector512.Count; + for (; index <= oneVectorFromEnd; index += Vector512.Count) + { + (Vector512 low, Vector512 high) = Vector512.Widen(Vector512.LoadUnsafe(ref samplesBase, (nuint)index)); + sum += Vector512.Sum(low) + Vector512.Sum(high); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector256.Count; + for (; index <= oneVectorFromEnd; index += Vector256.Count) + { + (Vector256 low, Vector256 high) = Vector256.Widen(Vector256.LoadUnsafe(ref samplesBase, (nuint)index)); + sum += Vector256.Sum(low) + Vector256.Sum(high); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector128.Count; + for (; index <= oneVectorFromEnd; index += Vector128.Count) + { + (Vector128 low, Vector128 high) = Vector128.Widen(Vector128.LoadUnsafe(ref samplesBase, (nuint)index)); + sum += Vector128.Sum(low) + Vector128.Sum(high); + } + } + + for (; index < samples.Length; index++) + { + sum += Unsafe.Add(ref samplesBase, index); + } + + return sum; + } + + /// + /// Sums high-bit-depth neighboring samples using the widest available SIMD width. + /// + /// The samples to sum. + /// The exact sum. + private static int Sum(ReadOnlySpan samples) + { + ref short samplesBase = ref MemoryMarshal.GetReference(samples); + int sum = 0; + int index = 0; + + if (Vector512.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector512.Count; + for (; index <= oneVectorFromEnd; index += Vector512.Count) + { + (Vector512 low, Vector512 high) = Vector512.Widen(Vector512.LoadUnsafe(ref samplesBase, (nuint)index)); + sum += Vector512.Sum(low) + Vector512.Sum(high); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector256.Count; + for (; index <= oneVectorFromEnd; index += Vector256.Count) + { + (Vector256 low, Vector256 high) = Vector256.Widen(Vector256.LoadUnsafe(ref samplesBase, (nuint)index)); + sum += Vector256.Sum(low) + Vector256.Sum(high); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = samples.Length - Vector128.Count; + for (; index <= oneVectorFromEnd; index += Vector128.Count) + { + (Vector128 low, Vector128 high) = Vector128.Widen(Vector128.LoadUnsafe(ref samplesBase, (nuint)index)); + sum += Vector128.Sum(low) + Vector128.Sum(high); + } + } + + for (; index < samples.Length; index++) + { + sum += Unsafe.Add(ref samplesBase, index); + } + + return sum; + } + + /// + /// Sums 8-bit neighboring samples without hardware intrinsics. + /// + /// The samples to sum. + /// The exact sum. + private static int SumScalar(ReadOnlySpan samples) + { + int sum = 0; + foreach (byte sample in samples) + { + sum += sample; + } + + return sum; + } + + /// + /// Sums high-bit-depth neighboring samples without hardware intrinsics. + /// + /// The samples to sum. + /// The exact sum. + private static int SumScalar(ReadOnlySpan samples) + { + int sum = 0; + foreach (short sample in samples) + { + sum += sample; + } + + return sum; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcLeftPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcLeftPredictor.cs deleted file mode 100644 index d2ffbbaca8..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcLeftPredictor.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block from the rounded average of its available left neighboring samples. -/// -internal readonly struct Av1DcLeftPredictor : IAv1Predictor -{ - /// - /// The number of samples written to each destination row. - /// - private readonly uint blockWidth; - - /// - /// The number of left samples averaged and destination rows written. - /// - private readonly uint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1DcLeftPredictor(Size blockSize) - { - this.blockWidth = (uint)blockSize.Width; - this.blockHeight = (uint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1DcLeftPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (uint)transformSize.GetWidth(); - this.blockHeight = (uint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block from its left neighboring samples. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The unused top-neighbor buffer required by the common predictor signature. - /// The left neighboring samples. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1DcLeftPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - int sum = 0; - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte destinationRef = ref destination[0]; - for (uint i = 0; i < this.blockHeight; i++) - { - sum += Unsafe.Add(ref leftRef, i); - } - - // Adding half the sample count implements the normative nearest-integer DC average before division. - byte expectedDc = (byte)((sum + (this.blockHeight >> 1)) / this.blockHeight); - for (uint r = 0; r < this.blockHeight; r++) - { - Unsafe.InitBlock(ref destinationRef, expectedDc, this.blockWidth); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcPredictor.cs deleted file mode 100644 index 220975b65c..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcPredictor.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block from the rounded average of its top and left neighboring samples. -/// -internal readonly struct Av1DcPredictor : IAv1Predictor -{ - /// - /// The number of top samples averaged and samples written to each destination row. - /// - private readonly nuint blockWidth; - - /// - /// The number of left samples averaged and destination rows written. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1DcPredictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1DcPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block from its top and left neighboring samples. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1DcPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - int sum = 0; - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - uint count = (uint)(this.blockWidth + this.blockHeight); - uint width = (uint)this.blockWidth; - for (nuint i = 0; i < this.blockWidth; i++) - { - sum += Unsafe.Add(ref aboveRef, i); - } - - for (nuint i = 0; i < this.blockHeight; i++) - { - sum += Unsafe.Add(ref leftRef, i); - } - - // Adding half the combined edge count implements the normative nearest-integer DC average before division. - byte expectedDc = (byte)((sum + (count >> 1)) / count); - for (nuint r = 0; r < this.blockHeight; r++) - { - Unsafe.InitBlock(ref destinationRef, expectedDc, width); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcTopPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcTopPredictor.cs deleted file mode 100644 index a8e41b2cb4..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcTopPredictor.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block from the rounded average of its available top neighboring samples. -/// -internal readonly struct Av1DcTopPredictor : IAv1Predictor -{ - /// - /// The number of top samples averaged and samples written to each destination row. - /// - private readonly uint blockWidth; - - /// - /// The number of destination rows. - /// - private readonly uint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1DcTopPredictor(Size blockSize) - { - this.blockWidth = (uint)blockSize.Width; - this.blockHeight = (uint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1DcTopPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (uint)transformSize.GetWidth(); - this.blockHeight = (uint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block from its top neighboring samples. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The unused left-neighbor buffer required by the common predictor signature. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1DcTopPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - int sum = 0; - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - for (uint i = 0; i < this.blockWidth; i++) - { - sum += Unsafe.Add(ref aboveRef, i); - } - - // Adding half the sample count implements the normative nearest-integer DC average before division. - byte expectedDc = (byte)((sum + (this.blockWidth >> 1)) / this.blockWidth); - for (uint r = 0; r < this.blockHeight; r++) - { - Unsafe.InitBlock(ref destinationRef, expectedDc, this.blockWidth); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Simd.cs new file mode 100644 index 0000000000..eb4ed15c6b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Simd.cs @@ -0,0 +1,619 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Provides the SIMD projection kernels for AV1 directional intra prediction. +/// +internal static partial class Av1DirectionalIntraPredictor +{ + /// + /// Gets the indices of even bytes in one upsampled reference vector. + /// + private static Vector128 EvenByteIndices => Vector128.Create((byte)0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14); + + /// + /// Gets the indices of odd bytes in one upsampled reference vector. + /// + private static Vector128 OddByteIndices => Vector128.Create((byte)1, 3, 5, 7, 9, 11, 13, 15, 1, 3, 5, 7, 9, 11, 13, 15); + + /// + /// Gets the indices of even high-bit-depth samples in one upsampled reference vector. + /// + private static Vector128 EvenShortIndices => Vector128.Create((short)0, 2, 4, 6, 0, 2, 4, 6); + + /// + /// Gets the indices of odd high-bit-depth samples in one upsampled reference vector. + /// + private static Vector128 OddShortIndices => Vector128.Create((short)1, 3, 5, 7, 1, 3, 5, 7); + + /// + /// Predicts one 8-bit zone 1 block using contiguous SIMD projection rows. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// Whether the reference contains half-sample positions. + /// The Q8 projection derivative. + /// The block width. + /// The block height. + private static void PredictZone1(Span destination, int destinationStride, ReadOnlySpan above, bool upsample, int derivative, int width, int height) + { + int upsampleShift = upsample ? 1 : 0; + int maximumBasis = (width + height - 1) << upsampleShift; + int fractionBits = 6 - upsampleShift; + int projection = derivative; + + for (int row = 0; row < height; row++, projection += derivative) + { + int basis = projection >> fractionBits; + int weight = ((projection << upsampleShift) & 0x3F) >> 1; + InterpolateRow(destination.Slice(row * destinationStride, width), above, basis, weight, upsample, maximumBasis); + } + } + + /// + /// Predicts one high-bit-depth zone 1 block using contiguous SIMD projection rows. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// Whether the reference contains half-sample positions. + /// The Q8 projection derivative. + /// The block width. + /// The block height. + private static void PredictZone1(Span destination, int destinationStride, ReadOnlySpan above, bool upsample, int derivative, int width, int height) + { + int upsampleShift = upsample ? 1 : 0; + int maximumBasis = (width + height - 1) << upsampleShift; + int fractionBits = 6 - upsampleShift; + int projection = derivative; + + for (int row = 0; row < height; row++, projection += derivative) + { + int basis = projection >> fractionBits; + int weight = ((projection << upsampleShift) & 0x3F) >> 1; + InterpolateRow(destination.Slice(row * destinationStride, width), above, basis, weight, upsample, maximumBasis); + } + } + + /// + /// Predicts one 8-bit zone 2 block using vectorized left gathers and contiguous top projection rows. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// The projected left reference. + /// Whether the top reference contains half-sample positions. + /// Whether the left reference contains half-sample positions. + /// The horizontal Q8 derivative. + /// The vertical Q8 derivative. + /// The block width. + /// The block height. + private static void PredictZone2(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int dx, int dy, int width, int height) + { + int aboveShift = upsampleAbove ? 1 : 0; + int minimumTopBasis = -(1 << aboveShift); + int topFractionBits = 6 - aboveShift; + int topBasisIncrement = 1 << aboveShift; + int topProjection = -dx; + + for (int row = 0; row < height; row++, topProjection -= dx) + { + int topBasis = topProjection >> topFractionBits; + int leftCount = 0; + while (leftCount < width && topBasis < minimumTopBasis) + { + leftCount++; + topBasis += topBasisIncrement; + } + + Span destinationRow = destination.Slice(row * destinationStride, width); + int leftProjection = (row << 6) - dy; + InterpolateLeft(destinationRow[..leftCount], left, leftProjection, dy, upsampleLeft); + + if (leftCount < width) + { + int topWeight = ((topProjection << aboveShift) & 0x3F) >> 1; + InterpolateRow(destinationRow[leftCount..], above, topBasis, topWeight, upsampleAbove, int.MaxValue); + } + } + } + + /// + /// Predicts one high-bit-depth zone 2 block using vectorized left gathers and contiguous top projection rows. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// The projected left reference. + /// Whether the top reference contains half-sample positions. + /// Whether the left reference contains half-sample positions. + /// The horizontal Q8 derivative. + /// The vertical Q8 derivative. + /// The block width. + /// The block height. + private static void PredictZone2(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int dx, int dy, int width, int height) + { + int aboveShift = upsampleAbove ? 1 : 0; + int minimumTopBasis = -(1 << aboveShift); + int topFractionBits = 6 - aboveShift; + int topBasisIncrement = 1 << aboveShift; + int topProjection = -dx; + + for (int row = 0; row < height; row++, topProjection -= dx) + { + int topBasis = topProjection >> topFractionBits; + int leftCount = 0; + while (leftCount < width && topBasis < minimumTopBasis) + { + leftCount++; + topBasis += topBasisIncrement; + } + + Span destinationRow = destination.Slice(row * destinationStride, width); + int leftProjection = (row << 6) - dy; + InterpolateLeft(destinationRow[..leftCount], left, leftProjection, dy, upsampleLeft); + + if (leftCount < width) + { + int topWeight = ((topProjection << aboveShift) & 0x3F) >> 1; + InterpolateRow(destinationRow[leftCount..], above, topBasis, topWeight, upsampleAbove, int.MaxValue); + } + } + } + + /// + /// Interpolates one 8-bit projection row. + /// + /// The destination row. + /// The projected reference samples. + /// The first integral reference coordinate. + /// The right-sample interpolation weight. + /// Whether consecutive output samples advance two reference positions. + /// The final extended reference coordinate, or when the row cannot reach it. + private static void InterpolateRow(Span destination, ReadOnlySpan reference, int basis, int weight, bool upsample, int maximumBasis) + { + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref byte referenceBase = ref MemoryMarshal.GetReference(reference); + int basisIncrement = upsample ? 2 : 1; + int validCount = maximumBasis == int.MaxValue || basis >= maximumBasis + ? maximumBasis == int.MaxValue ? destination.Length : 0 + : Math.Min(destination.Length, ((maximumBasis - 1 - basis) / basisIncrement) + 1); + int index = 0; + + if (!upsample) + { + if (Vector512.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - Vector512.Count; + for (; index <= oneVectorFromEnd; index += Vector512.Count) + { + Vector512 left = Vector512.LoadUnsafe(ref referenceBase, (nuint)(basis + index)); + Vector512 right = Vector512.LoadUnsafe(ref referenceBase, (nuint)(basis + index + 1)); + Interpolate(left, right, weight).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - Vector256.Count; + for (; index <= oneVectorFromEnd; index += Vector256.Count) + { + Vector256 left = Vector256.LoadUnsafe(ref referenceBase, (nuint)(basis + index)); + Vector256 right = Vector256.LoadUnsafe(ref referenceBase, (nuint)(basis + index + 1)); + Interpolate(left, right, weight).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - Vector128.Count; + for (; index <= oneVectorFromEnd; index += Vector128.Count) + { + Vector128 left = Vector128.LoadUnsafe(ref referenceBase, (nuint)(basis + index)); + Vector128 right = Vector128.LoadUnsafe(ref referenceBase, (nuint)(basis + index + 1)); + Interpolate(left, right, weight).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + } + else if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - 8; + for (; index <= oneVectorFromEnd; index += 8) + { + Vector128 source = Vector128.LoadUnsafe(ref referenceBase, (nuint)(basis + (index * 2))); + + // ShuffleNative maps to byte-table lookup on AdvSimd and PSHUFB on x86. Eight output samples are + // gathered from sixteen half-sample positions without scalar lane construction. + Vector128 left = Vector128.ShuffleNative(source, EvenByteIndices); + Vector128 right = Vector128.ShuffleNative(source, OddByteIndices); + Vector128 prediction = Interpolate(left, right, weight); + Unsafe.As(ref Unsafe.Add(ref destinationBase, index)) = prediction.AsUInt64().GetElement(0); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - 4; + for (; index <= oneVectorFromEnd; index += 4) + { + int source = basis + (index * basisIncrement); + Vector128 left = Vector128.Create( + (int)Unsafe.Add(ref referenceBase, source), + Unsafe.Add(ref referenceBase, source + basisIncrement), + Unsafe.Add(ref referenceBase, source + (2 * basisIncrement)), + Unsafe.Add(ref referenceBase, source + (3 * basisIncrement))); + + Vector128 right = Vector128.Create( + (int)Unsafe.Add(ref referenceBase, source + 1), + Unsafe.Add(ref referenceBase, source + basisIncrement + 1), + Unsafe.Add(ref referenceBase, source + (2 * basisIncrement) + 1), + Unsafe.Add(ref referenceBase, source + (3 * basisIncrement) + 1)); + + StoreFourBytes(Interpolate(left, right, Vector128.Create(weight)), ref Unsafe.Add(ref destinationBase, index)); + } + } + + for (; index < validCount; index++) + { + int source = basis + (index * basisIncrement); + Unsafe.Add(ref destinationBase, index) = (byte)(((Unsafe.Add(ref referenceBase, source) * (32 - weight)) + (Unsafe.Add(ref referenceBase, source + 1) * weight) + 16) >> 5); + } + + if (validCount < destination.Length) + { + destination[validCount..].Fill(Unsafe.Add(ref referenceBase, maximumBasis)); + } + } + + /// + /// Interpolates one high-bit-depth projection row. + /// + /// The destination row. + /// The projected reference samples. + /// The first integral reference coordinate. + /// The right-sample interpolation weight. + /// Whether consecutive output samples advance two reference positions. + /// The final extended reference coordinate, or when the row cannot reach it. + private static void InterpolateRow(Span destination, ReadOnlySpan reference, int basis, int weight, bool upsample, int maximumBasis) + { + ref short destinationBase = ref MemoryMarshal.GetReference(destination); + ref short referenceBase = ref MemoryMarshal.GetReference(reference); + int basisIncrement = upsample ? 2 : 1; + int validCount = maximumBasis == int.MaxValue || basis >= maximumBasis + ? maximumBasis == int.MaxValue ? destination.Length : 0 + : Math.Min(destination.Length, ((maximumBasis - 1 - basis) / basisIncrement) + 1); + int index = 0; + + if (!upsample) + { + if (Vector512.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - Vector512.Count; + for (; index <= oneVectorFromEnd; index += Vector512.Count) + { + Vector512 left = Vector512.LoadUnsafe(ref referenceBase, (nuint)(basis + index)); + Vector512 right = Vector512.LoadUnsafe(ref referenceBase, (nuint)(basis + index + 1)); + Interpolate(left, right, weight).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - Vector256.Count; + for (; index <= oneVectorFromEnd; index += Vector256.Count) + { + Vector256 left = Vector256.LoadUnsafe(ref referenceBase, (nuint)(basis + index)); + Vector256 right = Vector256.LoadUnsafe(ref referenceBase, (nuint)(basis + index + 1)); + Interpolate(left, right, weight).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - Vector128.Count; + for (; index <= oneVectorFromEnd; index += Vector128.Count) + { + Vector128 left = Vector128.LoadUnsafe(ref referenceBase, (nuint)(basis + index)); + Vector128 right = Vector128.LoadUnsafe(ref referenceBase, (nuint)(basis + index + 1)); + Interpolate(left, right, weight).StoreUnsafe(ref destinationBase, (nuint)index); + } + } + } + else if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - 4; + for (; index <= oneVectorFromEnd; index += 4) + { + Vector128 source = Vector128.LoadUnsafe(ref referenceBase, (nuint)(basis + (index * 2))); + Vector128 left = Vector128.ShuffleNative(source, EvenShortIndices); + Vector128 right = Vector128.ShuffleNative(source, OddShortIndices); + Vector128 prediction = Interpolate(left, right, weight); + Unsafe.As(ref Unsafe.Add(ref destinationBase, index)) = prediction.AsUInt64().GetElement(0); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = validCount - 4; + for (; index <= oneVectorFromEnd; index += 4) + { + int source = basis + (index * basisIncrement); + Vector128 left = Vector128.Create( + (int)Unsafe.Add(ref referenceBase, source), + Unsafe.Add(ref referenceBase, source + basisIncrement), + Unsafe.Add(ref referenceBase, source + (2 * basisIncrement)), + Unsafe.Add(ref referenceBase, source + (3 * basisIncrement))); + + Vector128 right = Vector128.Create( + (int)Unsafe.Add(ref referenceBase, source + 1), + Unsafe.Add(ref referenceBase, source + basisIncrement + 1), + Unsafe.Add(ref referenceBase, source + (2 * basisIncrement) + 1), + Unsafe.Add(ref referenceBase, source + (3 * basisIncrement) + 1)); + + StoreFourShorts(Interpolate(left, right, Vector128.Create(weight)), ref Unsafe.Add(ref destinationBase, index)); + } + } + + for (; index < validCount; index++) + { + int source = basis + (index * basisIncrement); + Unsafe.Add(ref destinationBase, index) = (short)(((Unsafe.Add(ref referenceBase, source) * (32 - weight)) + (Unsafe.Add(ref referenceBase, source + 1) * weight) + 16) >> 5); + } + + if (validCount < destination.Length) + { + destination[validCount..].Fill(Unsafe.Add(ref referenceBase, maximumBasis)); + } + } + + /// + /// Interpolates the left-edge prefix of one 8-bit zone 2 row. + /// + /// The destination prefix. + /// The projected left reference. + /// The first Q6 left projection. + /// The Q8 derivative subtracted between columns. + /// Whether the left reference contains half-sample positions. + private static void InterpolateLeft(Span destination, ReadOnlySpan left, int projection, int derivative, bool upsample) + { + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref byte leftBase = ref MemoryMarshal.GetReference(left); + int upsampleShift = upsample ? 1 : 0; + int fractionBits = 6 - upsampleShift; + int index = 0; + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = destination.Length - 4; + for (; index <= oneVectorFromEnd; index += 4) + { + int projection0 = projection - (index * derivative); + int projection1 = projection0 - derivative; + int projection2 = projection1 - derivative; + int projection3 = projection2 - derivative; + int basis0 = projection0 >> fractionBits; + int basis1 = projection1 >> fractionBits; + int basis2 = projection2 >> fractionBits; + int basis3 = projection3 >> fractionBits; + Vector128 source0 = Vector128.Create((int)Unsafe.Add(ref leftBase, basis0), Unsafe.Add(ref leftBase, basis1), Unsafe.Add(ref leftBase, basis2), Unsafe.Add(ref leftBase, basis3)); + Vector128 source1 = Vector128.Create((int)Unsafe.Add(ref leftBase, basis0 + 1), Unsafe.Add(ref leftBase, basis1 + 1), Unsafe.Add(ref leftBase, basis2 + 1), Unsafe.Add(ref leftBase, basis3 + 1)); + Vector128 weights = Vector128.Create( + ((projection0 << upsampleShift) & 0x3F) >> 1, + ((projection1 << upsampleShift) & 0x3F) >> 1, + ((projection2 << upsampleShift) & 0x3F) >> 1, + ((projection3 << upsampleShift) & 0x3F) >> 1); + + StoreFourBytes(Interpolate(source0, source1, weights), ref Unsafe.Add(ref destinationBase, index)); + } + } + + for (; index < destination.Length; index++) + { + int currentProjection = projection - (index * derivative); + int basis = currentProjection >> fractionBits; + int weight = ((currentProjection << upsampleShift) & 0x3F) >> 1; + Unsafe.Add(ref destinationBase, index) = (byte)(((Unsafe.Add(ref leftBase, basis) * (32 - weight)) + (Unsafe.Add(ref leftBase, basis + 1) * weight) + 16) >> 5); + } + } + + /// + /// Interpolates the left-edge prefix of one high-bit-depth zone 2 row. + /// + /// The destination prefix. + /// The projected left reference. + /// The first Q6 left projection. + /// The Q8 derivative subtracted between columns. + /// Whether the left reference contains half-sample positions. + private static void InterpolateLeft(Span destination, ReadOnlySpan left, int projection, int derivative, bool upsample) + { + ref short destinationBase = ref MemoryMarshal.GetReference(destination); + ref short leftBase = ref MemoryMarshal.GetReference(left); + int upsampleShift = upsample ? 1 : 0; + int fractionBits = 6 - upsampleShift; + int index = 0; + + if (Vector128.IsHardwareAccelerated) + { + int oneVectorFromEnd = destination.Length - 4; + for (; index <= oneVectorFromEnd; index += 4) + { + int projection0 = projection - (index * derivative); + int projection1 = projection0 - derivative; + int projection2 = projection1 - derivative; + int projection3 = projection2 - derivative; + int basis0 = projection0 >> fractionBits; + int basis1 = projection1 >> fractionBits; + int basis2 = projection2 >> fractionBits; + int basis3 = projection3 >> fractionBits; + Vector128 source0 = Vector128.Create((int)Unsafe.Add(ref leftBase, basis0), Unsafe.Add(ref leftBase, basis1), Unsafe.Add(ref leftBase, basis2), Unsafe.Add(ref leftBase, basis3)); + Vector128 source1 = Vector128.Create((int)Unsafe.Add(ref leftBase, basis0 + 1), Unsafe.Add(ref leftBase, basis1 + 1), Unsafe.Add(ref leftBase, basis2 + 1), Unsafe.Add(ref leftBase, basis3 + 1)); + Vector128 weights = Vector128.Create( + ((projection0 << upsampleShift) & 0x3F) >> 1, + ((projection1 << upsampleShift) & 0x3F) >> 1, + ((projection2 << upsampleShift) & 0x3F) >> 1, + ((projection3 << upsampleShift) & 0x3F) >> 1); + + StoreFourShorts(Interpolate(source0, source1, weights), ref Unsafe.Add(ref destinationBase, index)); + } + } + + for (; index < destination.Length; index++) + { + int currentProjection = projection - (index * derivative); + int basis = currentProjection >> fractionBits; + int weight = ((currentProjection << upsampleShift) & 0x3F) >> 1; + Unsafe.Add(ref destinationBase, index) = (short)(((Unsafe.Add(ref leftBase, basis) * (32 - weight)) + (Unsafe.Add(ref leftBase, basis + 1) * weight) + 16) >> 5); + } + } + + /// + /// Interpolates sixty-four pairs of 8-bit references. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weight. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Interpolate(Vector512 left, Vector512 right, int weight) + { + (Vector512 leftLow, Vector512 leftHigh) = Vector512.Widen(left); + (Vector512 rightLow, Vector512 rightHigh) = Vector512.Widen(right); + Vector512 rounding = Vector512.Create((ushort)16); + Vector512 low = ((leftLow * (ushort)(32 - weight)) + (rightLow * (ushort)weight) + rounding) >> 5; + Vector512 high = ((leftHigh * (ushort)(32 - weight)) + (rightHigh * (ushort)weight) + rounding) >> 5; + return Vector512.Narrow(low, high); + } + + /// + /// Interpolates thirty-two pairs of 8-bit references. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weight. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Interpolate(Vector256 left, Vector256 right, int weight) + { + (Vector256 leftLow, Vector256 leftHigh) = Vector256.Widen(left); + (Vector256 rightLow, Vector256 rightHigh) = Vector256.Widen(right); + Vector256 rounding = Vector256.Create((ushort)16); + Vector256 low = ((leftLow * (ushort)(32 - weight)) + (rightLow * (ushort)weight) + rounding) >> 5; + Vector256 high = ((leftHigh * (ushort)(32 - weight)) + (rightHigh * (ushort)weight) + rounding) >> 5; + return Vector256.Narrow(low, high); + } + + /// + /// Interpolates sixteen pairs of 8-bit references. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weight. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Interpolate(Vector128 left, Vector128 right, int weight) + { + (Vector128 leftLow, Vector128 leftHigh) = Vector128.Widen(left); + (Vector128 rightLow, Vector128 rightHigh) = Vector128.Widen(right); + Vector128 rounding = Vector128.Create((ushort)16); + Vector128 low = ((leftLow * (ushort)(32 - weight)) + (rightLow * (ushort)weight) + rounding) >> 5; + Vector128 high = ((leftHigh * (ushort)(32 - weight)) + (rightHigh * (ushort)weight) + rounding) >> 5; + return Vector128.Narrow(low, high); + } + + /// + /// Interpolates thirty-two pairs of high-bit-depth references. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weight. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Interpolate(Vector512 left, Vector512 right, int weight) + { + (Vector512 leftLow, Vector512 leftHigh) = Vector512.Widen(left); + (Vector512 rightLow, Vector512 rightHigh) = Vector512.Widen(right); + Vector512 rounding = Vector512.Create(16); + Vector512 low = ((leftLow * (32 - weight)) + (rightLow * weight) + rounding) >> 5; + Vector512 high = ((leftHigh * (32 - weight)) + (rightHigh * weight) + rounding) >> 5; + return Vector512.Narrow(low, high); + } + + /// + /// Interpolates sixteen pairs of high-bit-depth references. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weight. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Interpolate(Vector256 left, Vector256 right, int weight) + { + (Vector256 leftLow, Vector256 leftHigh) = Vector256.Widen(left); + (Vector256 rightLow, Vector256 rightHigh) = Vector256.Widen(right); + Vector256 rounding = Vector256.Create(16); + Vector256 low = ((leftLow * (32 - weight)) + (rightLow * weight) + rounding) >> 5; + Vector256 high = ((leftHigh * (32 - weight)) + (rightHigh * weight) + rounding) >> 5; + return Vector256.Narrow(low, high); + } + + /// + /// Interpolates eight pairs of high-bit-depth references. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weight. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Interpolate(Vector128 left, Vector128 right, int weight) + { + (Vector128 leftLow, Vector128 leftHigh) = Vector128.Widen(left); + (Vector128 rightLow, Vector128 rightHigh) = Vector128.Widen(right); + Vector128 rounding = Vector128.Create(16); + Vector128 low = ((leftLow * (32 - weight)) + (rightLow * weight) + rounding) >> 5; + Vector128 high = ((leftHigh * (32 - weight)) + (rightHigh * weight) + rounding) >> 5; + return Vector128.Narrow(low, high); + } + + /// + /// Interpolates four widened reference pairs with independent weights. + /// + /// The left interpolation samples. + /// The right interpolation samples. + /// The right-sample weights. + /// The rounded interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Interpolate(Vector128 left, Vector128 right, Vector128 weights) + => ((left * (Vector128.Create(32) - weights)) + (right * weights) + Vector128.Create(16)) >> 5; + + /// + /// Stores four widened predictions as packed 8-bit samples. + /// + /// The widened predictions. + /// The first destination sample. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreFourBytes(Vector128 prediction, ref byte destination) + { + Vector128 narrowed16 = Vector128.Narrow(prediction.AsUInt32(), Vector128.Zero); + Vector128 narrowed8 = Vector128.Narrow(narrowed16, Vector128.Zero); + Unsafe.As(ref destination) = narrowed8.AsUInt32().GetElement(0); + } + + /// + /// Stores four widened predictions as packed high-bit-depth samples. + /// + /// The widened predictions. + /// The first destination sample. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreFourShorts(Vector128 prediction, ref short destination) + { + Vector128 narrowed = Vector128.Narrow(prediction, Vector128.Zero); + Unsafe.As(ref destination) = narrowed.AsUInt64().GetElement(0); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Transpose.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Transpose.cs new file mode 100644 index 0000000000..c980d833f1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Transpose.cs @@ -0,0 +1,313 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Provides SIMD block transposition for zone 3 directional prediction. +/// +internal static partial class Av1DirectionalIntraPredictor +{ + /// + /// Transposes a contiguous 8-bit prediction block into the reconstructed destination. + /// + /// The contiguous source block. + /// The destination block origin. + /// The source row width. + /// The number of source rows. + /// The destination row stride. + private static void Transpose(ReadOnlySpan source, Span destination, int sourceWidth, int sourceHeight, int destinationStride) + { + if (Vector128.IsHardwareAccelerated) + { + int tileSize = sourceWidth >= 8 && sourceHeight >= 8 ? 8 : 4; + for (int y = 0; y < sourceHeight; y += tileSize) + { + for (int x = 0; x < sourceWidth; x += tileSize) + { + if (tileSize == 8) + { + Transpose8x8(source, destination, sourceWidth, destinationStride, x, y); + } + else + { + Transpose4x4(source, destination, sourceWidth, destinationStride, x, y); + } + } + } + + return; + } + + for (int y = 0; y < sourceHeight; y++) + { + for (int x = 0; x < sourceWidth; x++) + { + destination[(x * destinationStride) + y] = source[(y * sourceWidth) + x]; + } + } + } + + /// + /// Transposes a contiguous high-bit-depth prediction block into the reconstructed destination. + /// + /// The contiguous source block. + /// The destination block origin. + /// The source row width. + /// The number of source rows. + /// The destination row stride. + private static void Transpose(ReadOnlySpan source, Span destination, int sourceWidth, int sourceHeight, int destinationStride) + { + if (Vector128.IsHardwareAccelerated) + { + int tileSize = sourceWidth >= 8 && sourceHeight >= 8 ? 8 : 4; + for (int y = 0; y < sourceHeight; y += tileSize) + { + for (int x = 0; x < sourceWidth; x += tileSize) + { + if (tileSize == 8) + { + Transpose8x8(source, destination, sourceWidth, destinationStride, x, y); + } + else + { + Transpose4x4(source, destination, sourceWidth, destinationStride, x, y); + } + } + } + + return; + } + + for (int y = 0; y < sourceHeight; y++) + { + for (int x = 0; x < sourceWidth; x++) + { + destination[(x * destinationStride) + y] = source[(y * sourceWidth) + x]; + } + } + } + + /// + /// Transposes one eight-by-eight tile of 8-bit prediction samples. + /// + /// The contiguous source block. + /// The destination block origin. + /// The source row stride. + /// The destination row stride. + /// The tile X coordinate. + /// The tile Y coordinate. + private static void Transpose8x8(ReadOnlySpan source, Span destination, int sourceStride, int destinationStride, int x, int y) + { + ref byte sourceBase = ref MemoryMarshal.GetReference(source); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + Vector128 row0 = LoadEightBytes(ref sourceBase, ((y + 0) * sourceStride) + x); + Vector128 row1 = LoadEightBytes(ref sourceBase, ((y + 1) * sourceStride) + x); + Vector128 row2 = LoadEightBytes(ref sourceBase, ((y + 2) * sourceStride) + x); + Vector128 row3 = LoadEightBytes(ref sourceBase, ((y + 3) * sourceStride) + x); + Vector128 row4 = LoadEightBytes(ref sourceBase, ((y + 4) * sourceStride) + x); + Vector128 row5 = LoadEightBytes(ref sourceBase, ((y + 5) * sourceStride) + x); + Vector128 row6 = LoadEightBytes(ref sourceBase, ((y + 6) * sourceStride) + x); + Vector128 row7 = LoadEightBytes(ref sourceBase, ((y + 7) * sourceStride) + x); + + // Three unpack stages exchange one, two, then four byte coordinates. Each final vector contains two + // complete source columns, which are written as two contiguous eight-byte destination rows. + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair2 = Vector128_.UnpackLow(row2, row3); + Vector128 pair4 = Vector128_.UnpackLow(row4, row5); + Vector128 pair6 = Vector128_.UnpackLow(row6, row7); + Vector128 quad0 = Vector128_.UnpackLow(pair0.AsInt16(), pair2.AsInt16()); + Vector128 quad1 = Vector128_.UnpackHigh(pair0.AsInt16(), pair2.AsInt16()); + Vector128 quad4 = Vector128_.UnpackLow(pair4.AsInt16(), pair6.AsInt16()); + Vector128 quad5 = Vector128_.UnpackHigh(pair4.AsInt16(), pair6.AsInt16()); + Vector128 columns01 = Vector128_.UnpackLow(quad0.AsInt32(), quad4.AsInt32()); + Vector128 columns23 = Vector128_.UnpackHigh(quad0.AsInt32(), quad4.AsInt32()); + Vector128 columns45 = Vector128_.UnpackLow(quad1.AsInt32(), quad5.AsInt32()); + Vector128 columns67 = Vector128_.UnpackHigh(quad1.AsInt32(), quad5.AsInt32()); + + StoreEightBytes(columns01.AsUInt64().GetElement(0), ref destinationBase, ((x + 0) * destinationStride) + y); + StoreEightBytes(columns01.AsUInt64().GetElement(1), ref destinationBase, ((x + 1) * destinationStride) + y); + StoreEightBytes(columns23.AsUInt64().GetElement(0), ref destinationBase, ((x + 2) * destinationStride) + y); + StoreEightBytes(columns23.AsUInt64().GetElement(1), ref destinationBase, ((x + 3) * destinationStride) + y); + StoreEightBytes(columns45.AsUInt64().GetElement(0), ref destinationBase, ((x + 4) * destinationStride) + y); + StoreEightBytes(columns45.AsUInt64().GetElement(1), ref destinationBase, ((x + 5) * destinationStride) + y); + StoreEightBytes(columns67.AsUInt64().GetElement(0), ref destinationBase, ((x + 6) * destinationStride) + y); + StoreEightBytes(columns67.AsUInt64().GetElement(1), ref destinationBase, ((x + 7) * destinationStride) + y); + } + + /// + /// Transposes one four-by-four tile of 8-bit prediction samples. + /// + /// The contiguous source block. + /// The destination block origin. + /// The source row stride. + /// The destination row stride. + /// The tile X coordinate. + /// The tile Y coordinate. + private static void Transpose4x4(ReadOnlySpan source, Span destination, int sourceStride, int destinationStride, int x, int y) + { + ref byte sourceBase = ref MemoryMarshal.GetReference(source); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + Vector128 row0 = LoadFourBytes(ref sourceBase, ((y + 0) * sourceStride) + x); + Vector128 row1 = LoadFourBytes(ref sourceBase, ((y + 1) * sourceStride) + x); + Vector128 row2 = LoadFourBytes(ref sourceBase, ((y + 2) * sourceStride) + x); + Vector128 row3 = LoadFourBytes(ref sourceBase, ((y + 3) * sourceStride) + x); + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair1 = Vector128_.UnpackLow(row2, row3); + Vector128 columns = Vector128_.UnpackLow(pair0.AsInt16(), pair1.AsInt16()); + + Vector128 packedColumns = columns.AsUInt32(); + StoreFourBytes(packedColumns.GetElement(0), ref destinationBase, ((x + 0) * destinationStride) + y); + StoreFourBytes(packedColumns.GetElement(1), ref destinationBase, ((x + 1) * destinationStride) + y); + StoreFourBytes(packedColumns.GetElement(2), ref destinationBase, ((x + 2) * destinationStride) + y); + StoreFourBytes(packedColumns.GetElement(3), ref destinationBase, ((x + 3) * destinationStride) + y); + } + + /// + /// Transposes one eight-by-eight tile of high-bit-depth prediction samples. + /// + /// The contiguous source block. + /// The destination block origin. + /// The source row stride. + /// The destination row stride. + /// The tile X coordinate. + /// The tile Y coordinate. + private static void Transpose8x8(ReadOnlySpan source, Span destination, int sourceStride, int destinationStride, int x, int y) + { + ref short sourceBase = ref MemoryMarshal.GetReference(source); + ref short destinationBase = ref MemoryMarshal.GetReference(destination); + Vector128 row0 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 0) * sourceStride) + x)); + Vector128 row1 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 1) * sourceStride) + x)); + Vector128 row2 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 2) * sourceStride) + x)); + Vector128 row3 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 3) * sourceStride) + x)); + Vector128 row4 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 4) * sourceStride) + x)); + Vector128 row5 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 5) * sourceStride) + x)); + Vector128 row6 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 6) * sourceStride) + x)); + Vector128 row7 = Vector128.LoadUnsafe(ref sourceBase, (nuint)(((y + 7) * sourceStride) + x)); + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair1 = Vector128_.UnpackHigh(row0, row1); + Vector128 pair2 = Vector128_.UnpackLow(row2, row3); + Vector128 pair3 = Vector128_.UnpackHigh(row2, row3); + Vector128 pair4 = Vector128_.UnpackLow(row4, row5); + Vector128 pair5 = Vector128_.UnpackHigh(row4, row5); + Vector128 pair6 = Vector128_.UnpackLow(row6, row7); + Vector128 pair7 = Vector128_.UnpackHigh(row6, row7); + Vector128 quad0 = Vector128_.UnpackLow(pair0.AsInt32(), pair2.AsInt32()); + Vector128 quad1 = Vector128_.UnpackHigh(pair0.AsInt32(), pair2.AsInt32()); + Vector128 quad2 = Vector128_.UnpackLow(pair1.AsInt32(), pair3.AsInt32()); + Vector128 quad3 = Vector128_.UnpackHigh(pair1.AsInt32(), pair3.AsInt32()); + Vector128 quad4 = Vector128_.UnpackLow(pair4.AsInt32(), pair6.AsInt32()); + Vector128 quad5 = Vector128_.UnpackHigh(pair4.AsInt32(), pair6.AsInt32()); + Vector128 quad6 = Vector128_.UnpackLow(pair5.AsInt32(), pair7.AsInt32()); + Vector128 quad7 = Vector128_.UnpackHigh(pair5.AsInt32(), pair7.AsInt32()); + Vector128 column0 = Vector128_.UnpackLow(quad0.AsInt64(), quad4.AsInt64()).AsInt16(); + Vector128 column1 = Vector128_.UnpackHigh(quad0.AsInt64(), quad4.AsInt64()).AsInt16(); + Vector128 column2 = Vector128_.UnpackLow(quad1.AsInt64(), quad5.AsInt64()).AsInt16(); + Vector128 column3 = Vector128_.UnpackHigh(quad1.AsInt64(), quad5.AsInt64()).AsInt16(); + Vector128 column4 = Vector128_.UnpackLow(quad2.AsInt64(), quad6.AsInt64()).AsInt16(); + Vector128 column5 = Vector128_.UnpackHigh(quad2.AsInt64(), quad6.AsInt64()).AsInt16(); + Vector128 column6 = Vector128_.UnpackLow(quad3.AsInt64(), quad7.AsInt64()).AsInt16(); + Vector128 column7 = Vector128_.UnpackHigh(quad3.AsInt64(), quad7.AsInt64()).AsInt16(); + column0.StoreUnsafe(ref destinationBase, (nuint)(((x + 0) * destinationStride) + y)); + column1.StoreUnsafe(ref destinationBase, (nuint)(((x + 1) * destinationStride) + y)); + column2.StoreUnsafe(ref destinationBase, (nuint)(((x + 2) * destinationStride) + y)); + column3.StoreUnsafe(ref destinationBase, (nuint)(((x + 3) * destinationStride) + y)); + column4.StoreUnsafe(ref destinationBase, (nuint)(((x + 4) * destinationStride) + y)); + column5.StoreUnsafe(ref destinationBase, (nuint)(((x + 5) * destinationStride) + y)); + column6.StoreUnsafe(ref destinationBase, (nuint)(((x + 6) * destinationStride) + y)); + column7.StoreUnsafe(ref destinationBase, (nuint)(((x + 7) * destinationStride) + y)); + } + + /// + /// Transposes one four-by-four tile of high-bit-depth prediction samples. + /// + /// The contiguous source block. + /// The destination block origin. + /// The source row stride. + /// The destination row stride. + /// The tile X coordinate. + /// The tile Y coordinate. + private static void Transpose4x4(ReadOnlySpan source, Span destination, int sourceStride, int destinationStride, int x, int y) + { + ref short sourceBase = ref MemoryMarshal.GetReference(source); + ref short destinationBase = ref MemoryMarshal.GetReference(destination); + Vector128 row0 = LoadFourShorts(ref sourceBase, ((y + 0) * sourceStride) + x); + Vector128 row1 = LoadFourShorts(ref sourceBase, ((y + 1) * sourceStride) + x); + Vector128 row2 = LoadFourShorts(ref sourceBase, ((y + 2) * sourceStride) + x); + Vector128 row3 = LoadFourShorts(ref sourceBase, ((y + 3) * sourceStride) + x); + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair1 = Vector128_.UnpackLow(row2, row3); + Vector128 columns01 = Vector128_.UnpackLow(pair0.AsInt32(), pair1.AsInt32()); + Vector128 columns23 = Vector128_.UnpackHigh(pair0.AsInt32(), pair1.AsInt32()); + + StoreFourShorts(columns01.AsUInt64().GetElement(0), ref destinationBase, ((x + 0) * destinationStride) + y); + StoreFourShorts(columns01.AsUInt64().GetElement(1), ref destinationBase, ((x + 1) * destinationStride) + y); + StoreFourShorts(columns23.AsUInt64().GetElement(0), ref destinationBase, ((x + 2) * destinationStride) + y); + StoreFourShorts(columns23.AsUInt64().GetElement(1), ref destinationBase, ((x + 3) * destinationStride) + y); + } + + /// + /// Loads eight bytes into the lower half of a vector without reading past a tile row. + /// + /// The source buffer origin. + /// The source offset. + /// The loaded bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadEightBytes(ref byte source, int offset) + => Vector128.Create(Unsafe.As(ref Unsafe.Add(ref source, offset)), 0UL).AsByte(); + + /// + /// Loads four bytes into the low vector lanes without reading past a tile row. + /// + /// The source buffer origin. + /// The source offset. + /// The loaded bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadFourBytes(ref byte source, int offset) + => Vector128.Create(Unsafe.As(ref Unsafe.Add(ref source, offset)), 0U, 0U, 0U).AsByte(); + + /// + /// Loads four high-bit-depth samples into the lower half of a vector without reading past a tile row. + /// + /// The source buffer origin. + /// The source offset. + /// The loaded samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadFourShorts(ref short source, int offset) + => Vector128.Create(Unsafe.As(ref Unsafe.Add(ref source, offset)), 0UL).AsInt16(); + + /// + /// Stores the lower eight bytes of a vector. + /// + /// The packed bytes. + /// The destination buffer origin. + /// The destination offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreEightBytes(ulong source, ref byte destination, int offset) + => Unsafe.As(ref Unsafe.Add(ref destination, offset)) = source; + + /// + /// Stores four bytes from the low vector lanes. + /// + /// The packed bytes. + /// The destination buffer origin. + /// The destination offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreFourBytes(uint source, ref byte destination, int offset) + => Unsafe.As(ref Unsafe.Add(ref destination, offset)) = source; + + /// + /// Stores four high-bit-depth samples from the low vector lanes. + /// + /// The packed samples. + /// The destination buffer origin. + /// The destination offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreFourShorts(ulong source, ref short destination, int offset) + => Unsafe.As(ref Unsafe.Add(ref destination, offset)) = source; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.cs new file mode 100644 index 0000000000..1f07788da4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.cs @@ -0,0 +1,443 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Reconstructs AV1 directional intra-prediction blocks from prepared neighboring samples. +/// +/// +/// The three projection zones implement the directional prediction process in section 7.11.2.4 of the AV1 specification. +/// +internal static partial class Av1DirectionalIntraPredictor +{ + /// + /// The largest number of samples required to transpose a directional prediction block. + /// + public const int ScratchLength = 64 * 64; + + /// + /// Gets the Q8 directional derivatives indexed by acute prediction angle. + /// + private static ReadOnlySpan DirectionalIntraDerivative => + [ + + // Zero entries represent angles which AV1 never signals. Direct indexing avoids a search or division in + // each directional block while retaining the exact fixed-point projections from the normative table. + 0, 0, 0, 1023, 0, 0, 547, 0, 0, 372, 0, 0, 0, 0, 273, 0, 0, 215, 0, 0, 178, 0, 0, + 151, 0, 0, 132, 0, 0, 116, 0, 0, 102, 0, 0, 0, 90, 0, 0, 80, 0, 0, 71, 0, 0, 64, 0, 0, + 57, 0, 0, 51, 0, 0, 45, 0, 0, 0, 40, 0, 0, 35, 0, 0, 31, 0, 0, 27, 0, 0, 23, 0, 0, + 19, 0, 0, 15, 0, 0, 0, 0, 11, 0, 0, 7, 0, 0, 3, 0, 0, + ]; + + /// + /// Predicts an 8-bit directional block using the widest available SIMD path. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The predicted block dimensions. + /// The prepared top reference, including any required extension. + /// The prepared left reference, including any required extension. + /// Whether the top edge contains half-sample positions. + /// Whether the left edge contains half-sample positions. + /// The adjusted prediction angle. + /// The caller-owned block transposition workspace. + public static void Predict(Span destination, int destinationStride, Av1TransformSize transformSize, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int angle, Span scratch) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + + if (angle is > 0 and < 90) + { + PredictZone1(destination, destinationStride, above, upsampleAbove, GetDeltaX(angle), width, height); + } + else if (angle is > 90 and < 180) + { + PredictZone2(destination, destinationStride, above, left, upsampleAbove, upsampleLeft, GetDeltaX(angle), GetDeltaY(angle), width, height); + } + else if (angle is > 180 and < 270) + { + // libaom computes zone 3 as a zone 1 block with swapped dimensions, then transposes it. This preserves + // contiguous reference reads and destination stores in both hot stages instead of scattering columns. + Span transposed = scratch[..(width * height)]; + PredictZone1(transposed, height, left, upsampleLeft, GetDeltaY(angle), height, width); + Transpose(transposed, destination, height, width, destinationStride); + } + else + { + Av1PredictionMode mode = angle == 90 ? Av1PredictionMode.Vertical : Av1PredictionMode.Horizontal; + Av1IntraPredictorBase.GetPredictor(mode).Predict(destination, destinationStride, above, left, width, height); + } + } + + /// + /// Predicts a high-bit-depth directional block using the widest available SIMD path. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The predicted block dimensions. + /// The prepared top reference, including any required extension. + /// The prepared left reference, including any required extension. + /// Whether the top edge contains half-sample positions. + /// Whether the left edge contains half-sample positions. + /// The adjusted prediction angle. + /// The caller-owned block transposition workspace. + public static void Predict(Span destination, int destinationStride, Av1TransformSize transformSize, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int angle, Span scratch) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + + if (angle is > 0 and < 90) + { + PredictZone1(destination, destinationStride, above, upsampleAbove, GetDeltaX(angle), width, height); + } + else if (angle is > 90 and < 180) + { + PredictZone2(destination, destinationStride, above, left, upsampleAbove, upsampleLeft, GetDeltaX(angle), GetDeltaY(angle), width, height); + } + else if (angle is > 180 and < 270) + { + Span transposed = scratch[..(width * height)]; + PredictZone1(transposed, height, left, upsampleLeft, GetDeltaY(angle), height, width); + Transpose(transposed, destination, height, width, destinationStride); + } + else + { + Av1PredictionMode mode = angle == 90 ? Av1PredictionMode.Vertical : Av1PredictionMode.Horizontal; + Av1IntraPredictorBase.GetPredictor(mode).Predict(destination, destinationStride, above, left, width, height); + } + } + + /// + /// Predicts an 8-bit directional block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The predicted block dimensions. + /// The prepared top reference, including any required extension. + /// The prepared left reference, including any required extension. + /// Whether the top edge contains half-sample positions. + /// Whether the left edge contains half-sample positions. + /// The adjusted prediction angle. + public static void PredictScalar(Span destination, int destinationStride, Av1TransformSize transformSize, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int angle) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + + if (angle is > 0 and < 90) + { + PredictZone1Scalar(destination, destinationStride, above, upsampleAbove, GetDeltaX(angle), width, height); + } + else if (angle is > 90 and < 180) + { + PredictZone2Scalar(destination, destinationStride, above, left, upsampleAbove, upsampleLeft, GetDeltaX(angle), GetDeltaY(angle), width, height); + } + else if (angle is > 180 and < 270) + { + PredictZone3Scalar(destination, destinationStride, left, upsampleLeft, GetDeltaY(angle), width, height); + } + else + { + Av1PredictionMode mode = angle == 90 ? Av1PredictionMode.Vertical : Av1PredictionMode.Horizontal; + Av1IntraPredictorBase.GetPredictor(mode).PredictScalar(destination, destinationStride, above, left, width, height); + } + } + + /// + /// Predicts a high-bit-depth directional block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The predicted block dimensions. + /// The prepared top reference, including any required extension. + /// The prepared left reference, including any required extension. + /// Whether the top edge contains half-sample positions. + /// Whether the left edge contains half-sample positions. + /// The adjusted prediction angle. + public static void PredictScalar(Span destination, int destinationStride, Av1TransformSize transformSize, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int angle) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + + if (angle is > 0 and < 90) + { + PredictZone1Scalar(destination, destinationStride, above, upsampleAbove, GetDeltaX(angle), width, height); + } + else if (angle is > 90 and < 180) + { + PredictZone2Scalar(destination, destinationStride, above, left, upsampleAbove, upsampleLeft, GetDeltaX(angle), GetDeltaY(angle), width, height); + } + else if (angle is > 180 and < 270) + { + PredictZone3Scalar(destination, destinationStride, left, upsampleLeft, GetDeltaY(angle), width, height); + } + else + { + Av1PredictionMode mode = angle == 90 ? Av1PredictionMode.Vertical : Av1PredictionMode.Horizontal; + Av1IntraPredictorBase.GetPredictor(mode).PredictScalar(destination, destinationStride, above, left, width, height); + } + } + + /// + /// Gets the horizontal Q8 projection derivative for an adjusted angle. + /// + /// The adjusted prediction angle. + /// The horizontal derivative, or one when the selected zone does not consume it. + public static int GetDeltaX(int angle) + => angle switch + { + > 0 and < 90 => DirectionalIntraDerivative[angle], + > 90 and < 180 => DirectionalIntraDerivative[180 - angle], + _ => 1, + }; + + /// + /// Gets the vertical Q8 projection derivative for an adjusted angle. + /// + /// The adjusted prediction angle. + /// The vertical derivative, or one when the selected zone does not consume it. + public static int GetDeltaY(int angle) + => angle switch + { + > 90 and < 180 => DirectionalIntraDerivative[angle - 90], + > 180 and < 270 => DirectionalIntraDerivative[270 - angle], + _ => 1, + }; + + /// + /// Predicts one 8-bit zone 1 block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// Whether the reference contains half-sample positions. + /// The Q8 projection derivative. + /// The block width. + /// The block height. + private static void PredictZone1Scalar(Span destination, int destinationStride, ReadOnlySpan above, bool upsample, int derivative, int width, int height) + { + int upsampleShift = upsample ? 1 : 0; + int maximumBasis = (width + height - 1) << upsampleShift; + int fractionBits = 6 - upsampleShift; + int basisIncrement = 1 << upsampleShift; + int projection = derivative; + ref byte aboveBase = ref Unsafe.AsRef(in above[0]); + + for (int row = 0; row < height; row++, projection += derivative) + { + int basis = projection >> fractionBits; + int weight = ((projection << upsampleShift) & 0x3F) >> 1; + ref byte destinationRow = ref destination[row * destinationStride]; + + for (int column = 0; column < width; column++, basis += basisIncrement) + { + Unsafe.Add(ref destinationRow, column) = basis < maximumBasis + ? (byte)(((Unsafe.Add(ref aboveBase, basis) * (32 - weight)) + (Unsafe.Add(ref aboveBase, basis + 1) * weight) + 16) >> 5) + : Unsafe.Add(ref aboveBase, maximumBasis); + } + } + } + + /// + /// Predicts one high-bit-depth zone 1 block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// Whether the reference contains half-sample positions. + /// The Q8 projection derivative. + /// The block width. + /// The block height. + private static void PredictZone1Scalar(Span destination, int destinationStride, ReadOnlySpan above, bool upsample, int derivative, int width, int height) + { + int upsampleShift = upsample ? 1 : 0; + int maximumBasis = (width + height - 1) << upsampleShift; + int fractionBits = 6 - upsampleShift; + int basisIncrement = 1 << upsampleShift; + int projection = derivative; + ref short aboveBase = ref Unsafe.AsRef(in above[0]); + + for (int row = 0; row < height; row++, projection += derivative) + { + int basis = projection >> fractionBits; + int weight = ((projection << upsampleShift) & 0x3F) >> 1; + ref short destinationRow = ref destination[row * destinationStride]; + + for (int column = 0; column < width; column++, basis += basisIncrement) + { + Unsafe.Add(ref destinationRow, column) = basis < maximumBasis + ? (short)(((Unsafe.Add(ref aboveBase, basis) * (32 - weight)) + (Unsafe.Add(ref aboveBase, basis + 1) * weight) + 16) >> 5) + : Unsafe.Add(ref aboveBase, maximumBasis); + } + } + } + + /// + /// Predicts one 8-bit zone 2 block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// The projected left reference. + /// Whether the top reference contains half-sample positions. + /// Whether the left reference contains half-sample positions. + /// The horizontal Q8 derivative. + /// The vertical Q8 derivative. + /// The block width. + /// The block height. + private static void PredictZone2Scalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int dx, int dy, int width, int height) + { + int aboveShift = upsampleAbove ? 1 : 0; + int leftShift = upsampleLeft ? 1 : 0; + int minimumTopBasis = -(1 << aboveShift); + int topFractionBits = 6 - aboveShift; + int leftFractionBits = 6 - leftShift; + int topBasisIncrement = 1 << aboveShift; + int topProjection = -dx; + ref byte aboveBase = ref Unsafe.AsRef(in above[0]); + ref byte leftBase = ref Unsafe.AsRef(in left[0]); + + for (int row = 0; row < height; row++, topProjection -= dx) + { + int topBasis = topProjection >> topFractionBits; + int topWeight = ((topProjection << aboveShift) & 0x3F) >> 1; + int leftProjection = (row << 6) - dy; + ref byte destinationRow = ref destination[row * destinationStride]; + + for (int column = 0; column < width; column++, topBasis += topBasisIncrement, leftProjection -= dy) + { + int prediction; + if (topBasis >= minimumTopBasis) + { + prediction = (Unsafe.Add(ref aboveBase, topBasis) * (32 - topWeight)) + (Unsafe.Add(ref aboveBase, topBasis + 1) * topWeight); + } + else + { + int leftBasis = leftProjection >> leftFractionBits; + int leftWeight = ((leftProjection << leftShift) & 0x3F) >> 1; + prediction = (Unsafe.Add(ref leftBase, leftBasis) * (32 - leftWeight)) + (Unsafe.Add(ref leftBase, leftBasis + 1) * leftWeight); + } + + Unsafe.Add(ref destinationRow, column) = (byte)((prediction + 16) >> 5); + } + } + } + + /// + /// Predicts one high-bit-depth zone 2 block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected top reference. + /// The projected left reference. + /// Whether the top reference contains half-sample positions. + /// Whether the left reference contains half-sample positions. + /// The horizontal Q8 derivative. + /// The vertical Q8 derivative. + /// The block width. + /// The block height. + private static void PredictZone2Scalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, bool upsampleAbove, bool upsampleLeft, int dx, int dy, int width, int height) + { + int aboveShift = upsampleAbove ? 1 : 0; + int leftShift = upsampleLeft ? 1 : 0; + int minimumTopBasis = -(1 << aboveShift); + int topFractionBits = 6 - aboveShift; + int leftFractionBits = 6 - leftShift; + int topBasisIncrement = 1 << aboveShift; + int topProjection = -dx; + ref short aboveBase = ref Unsafe.AsRef(in above[0]); + ref short leftBase = ref Unsafe.AsRef(in left[0]); + + for (int row = 0; row < height; row++, topProjection -= dx) + { + int topBasis = topProjection >> topFractionBits; + int topWeight = ((topProjection << aboveShift) & 0x3F) >> 1; + int leftProjection = (row << 6) - dy; + ref short destinationRow = ref destination[row * destinationStride]; + + for (int column = 0; column < width; column++, topBasis += topBasisIncrement, leftProjection -= dy) + { + int prediction; + if (topBasis >= minimumTopBasis) + { + prediction = (Unsafe.Add(ref aboveBase, topBasis) * (32 - topWeight)) + (Unsafe.Add(ref aboveBase, topBasis + 1) * topWeight); + } + else + { + int leftBasis = leftProjection >> leftFractionBits; + int leftWeight = ((leftProjection << leftShift) & 0x3F) >> 1; + prediction = (Unsafe.Add(ref leftBase, leftBasis) * (32 - leftWeight)) + (Unsafe.Add(ref leftBase, leftBasis + 1) * leftWeight); + } + + Unsafe.Add(ref destinationRow, column) = (short)((prediction + 16) >> 5); + } + } + } + + /// + /// Predicts one 8-bit zone 3 block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected left reference. + /// Whether the reference contains half-sample positions. + /// The Q8 projection derivative. + /// The block width. + /// The block height. + private static void PredictZone3Scalar(Span destination, int destinationStride, ReadOnlySpan left, bool upsample, int derivative, int width, int height) + { + int upsampleShift = upsample ? 1 : 0; + int maximumBasis = (width + height - 1) << upsampleShift; + int fractionBits = 6 - upsampleShift; + int basisIncrement = 1 << upsampleShift; + int projection = derivative; + ref byte leftBase = ref Unsafe.AsRef(in left[0]); + + for (int column = 0; column < width; column++, projection += derivative) + { + int basis = projection >> fractionBits; + int weight = ((projection << upsampleShift) & 0x3F) >> 1; + for (int row = 0; row < height; row++, basis += basisIncrement) + { + destination[(row * destinationStride) + column] = basis < maximumBasis + ? (byte)(((Unsafe.Add(ref leftBase, basis) * (32 - weight)) + (Unsafe.Add(ref leftBase, basis + 1) * weight) + 16) >> 5) + : Unsafe.Add(ref leftBase, maximumBasis); + } + } + } + + /// + /// Predicts one high-bit-depth zone 3 block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride. + /// The projected left reference. + /// Whether the reference contains half-sample positions. + /// The Q8 projection derivative. + /// The block width. + /// The block height. + private static void PredictZone3Scalar(Span destination, int destinationStride, ReadOnlySpan left, bool upsample, int derivative, int width, int height) + { + int upsampleShift = upsample ? 1 : 0; + int maximumBasis = (width + height - 1) << upsampleShift; + int fractionBits = 6 - upsampleShift; + int basisIncrement = 1 << upsampleShift; + int projection = derivative; + ref short leftBase = ref Unsafe.AsRef(in left[0]); + + for (int column = 0; column < width; column++, projection += derivative) + { + int basis = projection >> fractionBits; + int weight = ((projection << upsampleShift) & 0x3F) >> 1; + for (int row = 0; row < height; row++, basis += basisIncrement) + { + destination[(row * destinationStride) + column] = basis < maximumBasis + ? (short)(((Unsafe.Add(ref leftBase, basis) * (32 - weight)) + (Unsafe.Add(ref leftBase, basis + 1) * weight) + 16) >> 5) + : Unsafe.Add(ref leftBase, maximumBasis); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone1Predictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone1Predictor.cs deleted file mode 100644 index cfb2f4ba9e..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone1Predictor.cs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Produces 8-bit AV1 directional intra predictions for angles in zone 1. -/// -/// -/// Zone 1 projects the top reference samples into the block for prediction angles less than 90 degrees. -/// The scalar prediction follows the directional prediction process in section 7.11.2.4 of the AV1 specification. -/// -internal readonly struct Av1DirectionalZone1Predictor -{ - /// - /// The width of the prediction block in samples. - /// - private readonly nuint blockWidth; - - /// - /// The height of the prediction block in samples. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for the specified block dimensions. - /// - /// The dimensions of the prediction block. - public Av1DirectionalZone1Predictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for the specified transform size. - /// - /// The transform size that determines the prediction block dimensions. - public Av1DirectionalZone1Predictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Produces an 8-bit zone 1 directional prediction for a transform block. - /// - /// The transform size that determines the prediction block dimensions. - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The top reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// The horizontal projection derivative in Q6 precision. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, bool upsampleAbove, int dx) - => new Av1DirectionalZone1Predictor(transformSize).PredictScalar(destination, stride, above, upsampleAbove, dx); - - /// - /// Produces an 8-bit zone 1 directional prediction for this block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The top reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// The horizontal projection derivative in Q6 precision. - /// Corresponds to svt_av1_dr_prediction_z1_c in SVT-AV1. - public void PredictScalar(Span destination, nuint stride, Span above, bool upsample, int dx) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - int upsampleAbove = upsample ? 1 : 0; - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - int maxBasisX = (((int)this.blockWidth + (int)this.blockHeight) - 1) << upsampleAbove; - int fractionBitCount = 6 - upsampleAbove; - int basisIncrement = 1 << upsampleAbove; - int x = dx; - for (nuint r = 0; r < this.blockHeight; ++r) - { - // AV1 retains six fractional projection bits, or five after reference upsampling. - // The interpolation weights sum to 32 because the low projection bit is discarded. - int basis = x >> fractionBitCount, shift = ((x << upsampleAbove) & 0x3F) >> 1; - - if (basis >= maxBasisX) - { - for (nuint i = r; i < this.blockHeight; ++i) - { - Unsafe.InitBlock(ref destinationRef, Unsafe.Add(ref aboveRef, maxBasisX), (uint)this.blockWidth); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - - return; - } - - for (nuint c = 0; c < this.blockWidth; ++c) - { - if (basis < maxBasisX) - { - int val; - val = (Unsafe.Add(ref aboveRef, basis) * (32 - shift)) + (Unsafe.Add(ref aboveRef, basis + 1) * shift); - val = Av1Math.RoundPowerOf2(val, 5); - Unsafe.Add(ref destinationRef, c) = (byte)Av1Math.Clamp(val, 0, 255); - } - else - { - Unsafe.Add(ref destinationRef, c) = Unsafe.Add(ref aboveRef, maxBasisX); - } - - basis += basisIncrement; - } - - x += dx; - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone2Predictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone2Predictor.cs deleted file mode 100644 index 257e0615e1..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone2Predictor.cs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Produces 8-bit AV1 directional intra predictions for angles in zone 2. -/// -/// -/// Zone 2 projects both top and left reference samples into the block for prediction angles between 90 and 180 degrees. -/// The scalar prediction follows the directional prediction process in section 7.11.2.4 of the AV1 specification. -/// -internal readonly struct Av1DirectionalZone2Predictor -{ - /// - /// The width of the prediction block in samples. - /// - private readonly nuint blockWidth; - - /// - /// The height of the prediction block in samples. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for the specified block dimensions. - /// - /// The dimensions of the prediction block. - public Av1DirectionalZone2Predictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for the specified transform size. - /// - /// The transform size that determines the prediction block dimensions. - public Av1DirectionalZone2Predictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Produces an 8-bit zone 2 directional prediction for a transform block. - /// - /// The transform size that determines the prediction block dimensions. - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The top reference samples, including any required extension. - /// The left reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// A value indicating whether the left reference samples were upsampled. - /// The horizontal projection derivative in Q6 precision. - /// The vertical projection derivative in Q6 precision. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left, bool upsampleAbove, bool upsampleLeft, int dx, int dy) - => new Av1DirectionalZone2Predictor(transformSize).PredictScalar(destination, stride, above, left, upsampleAbove, upsampleLeft, dx, dy); - - /// - /// Produces an 8-bit zone 2 directional prediction for this block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The top reference samples, including any required extension. - /// The left reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// A value indicating whether the left reference samples were upsampled. - /// The horizontal projection derivative in Q6 precision. - /// The vertical projection derivative in Q6 precision. - /// Corresponds to svt_av1_dr_prediction_z2_c in SVT-AV1. - public void PredictScalar(Span destination, nuint stride, Span above, Span left, bool doUpsampleAbove, bool doUpsampleLeft, int dx, int dy) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - int upsampleAbove = doUpsampleAbove ? 1 : 0; - int upsampleLeft = doUpsampleLeft ? 1 : 0; - ref byte aboveRef = ref above[0]; - ref byte leftRef = ref left[0]; - ref byte destinationRef = ref destination[0]; - int minBasisX = -(1 << upsampleAbove); - int fractionBitCountX = 6 - upsampleAbove; - int fractionBitCountY = 6 - upsampleLeft; - int basisIncrementX = 1 << upsampleAbove; - int x = -dx; - for (nuint r = 0; r < this.blockHeight; ++r) - { - int val; - int base1 = x >> fractionBitCountX; - int y = ((int)r << 6) - dy; - for (nuint c = 0; c < this.blockWidth; ++c, base1 += basisIncrementX, y -= dy) - { - // A nonnegative top projection uses the above edge. Once the projection crosses - // the top-left corner, the same destination sample is projected from the left edge. - if (base1 >= minBasisX) - { - int shift1 = ((x * (1 << upsampleAbove)) & 0x3F) >> 1; - val = (Unsafe.Add(ref aboveRef, base1) * (32 - shift1)) + (Unsafe.Add(ref aboveRef, base1 + 1) * shift1); - val = Av1Math.RoundPowerOf2(val, 5); - } - else - { - int base2 = y >> fractionBitCountY; - Guard.MustBeGreaterThanOrEqualTo(base2, -(1 << upsampleLeft), nameof(base2)); - int shift2 = ((y * (1 << upsampleLeft)) & 0x3F) >> 1; - val = (Unsafe.Add(ref leftRef, base2) * (32 - shift2)) + (Unsafe.Add(ref leftRef, base2 + 1) * shift2); - val = Av1Math.RoundPowerOf2(val, 5); - } - - Unsafe.Add(ref destinationRef, c) = (byte)Av1Math.Clamp(val, 0, 255); - } - - x -= dx; - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone3Predictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone3Predictor.cs deleted file mode 100644 index 5caa1d81c0..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalZone3Predictor.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System; -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Produces 8-bit AV1 directional intra predictions for angles in zone 3. -/// -/// -/// Zone 3 projects the left reference samples into the block for prediction angles greater than 180 degrees. -/// The scalar prediction follows the directional prediction process in section 7.11.2.4 of the AV1 specification. -/// -internal readonly struct Av1DirectionalZone3Predictor -{ - /// - /// The width of the prediction block in samples. - /// - private readonly nuint blockWidth; - - /// - /// The height of the prediction block in samples. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for the specified block dimensions. - /// - /// The dimensions of the prediction block. - public Av1DirectionalZone3Predictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for the specified transform size. - /// - /// The transform size that determines the prediction block dimensions. - public Av1DirectionalZone3Predictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Produces an 8-bit zone 3 directional prediction for a transform block. - /// - /// The transform size that determines the prediction block dimensions. - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The left reference samples, including any required extension. - /// A value indicating whether the left reference samples were upsampled. - /// The horizontal projection derivative, which must be one in zone 3. - /// The vertical projection derivative in Q6 precision. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span left, bool upsampleLeft, int dx, int dy) - => new Av1DirectionalZone3Predictor(transformSize).PredictScalar(destination, stride, left, upsampleLeft, dx, dy); - - /// - /// Produces an 8-bit zone 3 directional prediction for this block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The left reference samples, including any required extension. - /// A value indicating whether the left reference samples were upsampled. - /// The horizontal projection derivative, which must be one in zone 3. - /// The vertical projection derivative in Q6 precision. - /// Corresponds to svt_av1_dr_prediction_z3_c in SVT-AV1. - public void PredictScalar(Span destination, nuint stride, Span left, bool upsample, int dx, int dy) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - int upsampleLeft = upsample ? 1 : 0; - ref byte leftRef = ref left[0]; - ref byte destinationRef = ref destination[0]; - Guard.IsTrue(dx == 1, nameof(dx), "Dx expected to be always equal to 1 for directional Zone 3 prediction."); - Guard.MustBeGreaterThan(dy, 0, nameof(dy)); - - int maxBasisY = ((int)this.blockWidth + (int)this.blockHeight - 1) << upsampleLeft; - int fractionBitCount = 6 - upsampleLeft; - int basisIncrement = 1 << upsampleLeft; - int y = dy; - for (nuint c = 0; c < this.blockWidth; ++c) - { - // Zone 3 is the transpose of zone 1: columns advance along the projected left edge, - // while rows advance through the reference samples for each destination column. - int basis = y >> fractionBitCount; - int shift = ((y << upsampleLeft) & 0x3F) >> 1; - - for (nuint r = 0; r < this.blockHeight; ++r) - { - if (basis < maxBasisY) - { - int val; - val = (Unsafe.Add(ref leftRef, basis) * (32 - shift)) + (Unsafe.Add(ref leftRef, basis + 1) * shift); - val = Av1Math.RoundPowerOf2(val, 5); - Unsafe.Add(ref destinationRef, (r * stride) + c) = (byte)Av1Math.Clamp(val, 0, 255); - } - else - { - for (; r < this.blockHeight; ++r) - { - Unsafe.Add(ref destinationRef, (r * stride) + c) = left[maxBasisY]; - } - - break; - } - - basis += basisIncrement; - } - - y += dy; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.DcOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.DcOperator.cs new file mode 100644 index 0000000000..c696afef76 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.DcOperator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Defines the AV1 DC filter-intra coefficient operator. + /// + internal readonly struct DcOperator : IAv1FilterIntraPredictionOperator + { + /// + public static Av1FilterIntraMode Mode => Av1FilterIntraMode.DC; + + /// + public static ReadOnlySpan Taps => + [ + -6, 10, 0, 0, 0, 12, 0, + -5, 2, 10, 0, 0, 9, 0, + -3, 1, 1, 10, 0, 7, 0, + -3, 1, 1, 2, 10, 5, 0, + -4, 6, 0, 0, 0, 2, 12, + -3, 2, 6, 0, 0, 2, 9, + -3, 2, 2, 6, 0, 2, 7, + -3, 1, 2, 2, 6, 3, 5, + ]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Directional157Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Directional157Operator.cs new file mode 100644 index 0000000000..5e8ddebcd1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Directional157Operator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Defines the AV1 157-degree directional filter-intra coefficient operator. + /// + internal readonly struct Directional157Operator : IAv1FilterIntraPredictionOperator + { + /// + public static Av1FilterIntraMode Mode => Av1FilterIntraMode.Directional157; + + /// + public static ReadOnlySpan Taps => + [ + -2, 8, 0, 0, 0, 10, 0, + -1, 3, 8, 0, 0, 6, 0, + -1, 2, 3, 8, 0, 4, 0, + 0, 1, 2, 3, 8, 2, 0, + -1, 4, 0, 0, 0, 3, 10, + -1, 3, 4, 0, 0, 4, 6, + -1, 2, 3, 4, 0, 4, 4, + -1, 2, 2, 3, 4, 3, 3, + ]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.HorizontalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.HorizontalOperator.cs new file mode 100644 index 0000000000..2b1da42d4e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.HorizontalOperator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Defines the AV1 horizontal filter-intra coefficient operator. + /// + internal readonly struct HorizontalOperator : IAv1FilterIntraPredictionOperator + { + /// + public static Av1FilterIntraMode Mode => Av1FilterIntraMode.Horizontal; + + /// + public static ReadOnlySpan Taps => + [ + -8, 8, 0, 0, 0, 16, 0, + -8, 0, 8, 0, 0, 16, 0, + -8, 0, 0, 8, 0, 16, 0, + -8, 0, 0, 0, 8, 16, 0, + -4, 4, 0, 0, 0, 0, 16, + -4, 0, 4, 0, 0, 0, 16, + -4, 0, 0, 4, 0, 0, 16, + -4, 0, 0, 0, 4, 0, 16, + ]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs new file mode 100644 index 0000000000..005ed33e26 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs @@ -0,0 +1,596 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Applies one closed filter-intra coefficient operator using the widest useful SIMD width. + /// + /// The filter-intra coefficient set. + internal sealed class Av1FilterIntraPredictor : Av1FilterIntraPredictorBase + where TOperator : struct, IAv1FilterIntraPredictionOperator + { + /// + public override Av1FilterIntraMode Mode => TOperator.Mode; + + /// + public override void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, Span scratch) + { + Span buffer = scratch[..ScratchLength]; + ref byte bufferBase = ref MemoryMarshal.GetReference(buffer); + ref byte aboveBase = ref MemoryMarshal.GetReference(above); + ref byte leftBase = ref MemoryMarshal.GetReference(left); + ref sbyte taps = ref MemoryMarshal.GetReference(TOperator.Taps); + Initialize(buffer, above, left, width, height, Unsafe.Subtract(ref aboveBase, 1)); + + if (Vector256.IsHardwareAccelerated) + { + Vector256 tap0 = CreateTapVector256(ref taps, 0); + Vector256 tap1 = CreateTapVector256(ref taps, 1); + Vector256 tap2 = CreateTapVector256(ref taps, 2); + Vector256 tap3 = CreateTapVector256(ref taps, 3); + Vector256 tap4 = CreateTapVector256(ref taps, 4); + Vector256 tap5 = CreateTapVector256(ref taps, 5); + Vector256 tap6 = CreateTapVector256(ref taps, 6); + Vector256 rounding = Vector256.Create(8); + Vector256 maximum = Vector256.Create(255); + + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + Vector256 prediction = Calculate( + tap0, + tap1, + tap2, + tap3, + tap4, + tap5, + tap6, + Unsafe.Add(ref bufferBase, sourceOffset), + Unsafe.Add(ref bufferBase, sourceOffset + 1), + Unsafe.Add(ref bufferBase, sourceOffset + 2), + Unsafe.Add(ref bufferBase, sourceOffset + 3), + Unsafe.Add(ref bufferBase, sourceOffset + 4), + Unsafe.Add(ref bufferBase, sourceOffset + BufferStride), + Unsafe.Add(ref bufferBase, sourceOffset + (2 * BufferStride))); + + prediction = Vector256.Clamp((prediction + rounding) >> 4, Vector256.Zero, maximum); + StoreEightBytes(prediction, ref bufferBase, (row * BufferStride) + column, ((row + 1) * BufferStride) + column); + } + } + } + else if (Vector128.IsHardwareAccelerated) + { + Vector128 tap0Low = CreateTapVector128(ref taps, 0, 0); + Vector128 tap1Low = CreateTapVector128(ref taps, 1, 0); + Vector128 tap2Low = CreateTapVector128(ref taps, 2, 0); + Vector128 tap3Low = CreateTapVector128(ref taps, 3, 0); + Vector128 tap4Low = CreateTapVector128(ref taps, 4, 0); + Vector128 tap5Low = CreateTapVector128(ref taps, 5, 0); + Vector128 tap6Low = CreateTapVector128(ref taps, 6, 0); + Vector128 tap0High = CreateTapVector128(ref taps, 0, 4); + Vector128 tap1High = CreateTapVector128(ref taps, 1, 4); + Vector128 tap2High = CreateTapVector128(ref taps, 2, 4); + Vector128 tap3High = CreateTapVector128(ref taps, 3, 4); + Vector128 tap4High = CreateTapVector128(ref taps, 4, 4); + Vector128 tap5High = CreateTapVector128(ref taps, 5, 4); + Vector128 tap6High = CreateTapVector128(ref taps, 6, 4); + Vector128 rounding = Vector128.Create(8); + Vector128 maximum = Vector128.Create(255); + + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + int p0 = Unsafe.Add(ref bufferBase, sourceOffset); + int p1 = Unsafe.Add(ref bufferBase, sourceOffset + 1); + int p2 = Unsafe.Add(ref bufferBase, sourceOffset + 2); + int p3 = Unsafe.Add(ref bufferBase, sourceOffset + 3); + int p4 = Unsafe.Add(ref bufferBase, sourceOffset + 4); + int p5 = Unsafe.Add(ref bufferBase, sourceOffset + BufferStride); + int p6 = Unsafe.Add(ref bufferBase, sourceOffset + (2 * BufferStride)); + Vector128 low = Calculate(tap0Low, tap1Low, tap2Low, tap3Low, tap4Low, tap5Low, tap6Low, p0, p1, p2, p3, p4, p5, p6); + Vector128 high = Calculate(tap0High, tap1High, tap2High, tap3High, tap4High, tap5High, tap6High, p0, p1, p2, p3, p4, p5, p6); + low = Vector128.Clamp((low + rounding) >> 4, Vector128.Zero, maximum); + high = Vector128.Clamp((high + rounding) >> 4, Vector128.Zero, maximum); + StoreFourBytes(low, ref bufferBase, (row * BufferStride) + column); + StoreFourBytes(high, ref bufferBase, ((row + 1) * BufferStride) + column); + } + } + } + else + { + PredictGroupsScalar(buffer, width, height, 255, ref taps); + } + + CopyToDestination(buffer, destination, destinationStride, width, height); + } + + /// + public override void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, int bitDepth, Span scratch) + { + Span buffer = scratch[..ScratchLength]; + ref short bufferBase = ref MemoryMarshal.GetReference(buffer); + ref short aboveBase = ref MemoryMarshal.GetReference(above); + ref sbyte taps = ref MemoryMarshal.GetReference(TOperator.Taps); + Initialize(buffer, above, left, width, height, Unsafe.Subtract(ref aboveBase, 1)); + int maximum = (1 << bitDepth) - 1; + + if (Vector256.IsHardwareAccelerated) + { + Vector256 tap0 = CreateTapVector256(ref taps, 0); + Vector256 tap1 = CreateTapVector256(ref taps, 1); + Vector256 tap2 = CreateTapVector256(ref taps, 2); + Vector256 tap3 = CreateTapVector256(ref taps, 3); + Vector256 tap4 = CreateTapVector256(ref taps, 4); + Vector256 tap5 = CreateTapVector256(ref taps, 5); + Vector256 tap6 = CreateTapVector256(ref taps, 6); + Vector256 rounding = Vector256.Create(8); + Vector256 maximumVector = Vector256.Create(maximum); + + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + Vector256 prediction = Calculate( + tap0, + tap1, + tap2, + tap3, + tap4, + tap5, + tap6, + Unsafe.Add(ref bufferBase, sourceOffset), + Unsafe.Add(ref bufferBase, sourceOffset + 1), + Unsafe.Add(ref bufferBase, sourceOffset + 2), + Unsafe.Add(ref bufferBase, sourceOffset + 3), + Unsafe.Add(ref bufferBase, sourceOffset + 4), + Unsafe.Add(ref bufferBase, sourceOffset + BufferStride), + Unsafe.Add(ref bufferBase, sourceOffset + (2 * BufferStride))); + + prediction = Vector256.Clamp((prediction + rounding) >> 4, Vector256.Zero, maximumVector); + StoreEightShorts(prediction, ref bufferBase, (row * BufferStride) + column, ((row + 1) * BufferStride) + column); + } + } + } + else if (Vector128.IsHardwareAccelerated) + { + Vector128 tap0Low = CreateTapVector128(ref taps, 0, 0); + Vector128 tap1Low = CreateTapVector128(ref taps, 1, 0); + Vector128 tap2Low = CreateTapVector128(ref taps, 2, 0); + Vector128 tap3Low = CreateTapVector128(ref taps, 3, 0); + Vector128 tap4Low = CreateTapVector128(ref taps, 4, 0); + Vector128 tap5Low = CreateTapVector128(ref taps, 5, 0); + Vector128 tap6Low = CreateTapVector128(ref taps, 6, 0); + Vector128 tap0High = CreateTapVector128(ref taps, 0, 4); + Vector128 tap1High = CreateTapVector128(ref taps, 1, 4); + Vector128 tap2High = CreateTapVector128(ref taps, 2, 4); + Vector128 tap3High = CreateTapVector128(ref taps, 3, 4); + Vector128 tap4High = CreateTapVector128(ref taps, 4, 4); + Vector128 tap5High = CreateTapVector128(ref taps, 5, 4); + Vector128 tap6High = CreateTapVector128(ref taps, 6, 4); + Vector128 rounding = Vector128.Create(8); + Vector128 maximumVector = Vector128.Create(maximum); + + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + int p0 = Unsafe.Add(ref bufferBase, sourceOffset); + int p1 = Unsafe.Add(ref bufferBase, sourceOffset + 1); + int p2 = Unsafe.Add(ref bufferBase, sourceOffset + 2); + int p3 = Unsafe.Add(ref bufferBase, sourceOffset + 3); + int p4 = Unsafe.Add(ref bufferBase, sourceOffset + 4); + int p5 = Unsafe.Add(ref bufferBase, sourceOffset + BufferStride); + int p6 = Unsafe.Add(ref bufferBase, sourceOffset + (2 * BufferStride)); + Vector128 low = Calculate(tap0Low, tap1Low, tap2Low, tap3Low, tap4Low, tap5Low, tap6Low, p0, p1, p2, p3, p4, p5, p6); + Vector128 high = Calculate(tap0High, tap1High, tap2High, tap3High, tap4High, tap5High, tap6High, p0, p1, p2, p3, p4, p5, p6); + low = Vector128.Clamp((low + rounding) >> 4, Vector128.Zero, maximumVector); + high = Vector128.Clamp((high + rounding) >> 4, Vector128.Zero, maximumVector); + StoreFourShorts(low, ref bufferBase, (row * BufferStride) + column); + StoreFourShorts(high, ref bufferBase, ((row + 1) * BufferStride) + column); + } + } + } + else + { + PredictGroupsScalar(buffer, width, height, maximum, ref taps); + } + + CopyToDestination(buffer, destination, destinationStride, width, height); + } + + /// + public override void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, Span scratch) + { + Span buffer = scratch[..ScratchLength]; + ref byte aboveBase = ref MemoryMarshal.GetReference(above); + ref sbyte taps = ref MemoryMarshal.GetReference(TOperator.Taps); + Initialize(buffer, above, left, width, height, Unsafe.Subtract(ref aboveBase, 1)); + PredictGroupsScalar(buffer, width, height, 255, ref taps); + CopyToDestinationScalar(buffer, destination, destinationStride, width, height); + } + + /// + public override void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, int bitDepth, Span scratch) + { + Span buffer = scratch[..ScratchLength]; + ref short aboveBase = ref MemoryMarshal.GetReference(above); + ref sbyte taps = ref MemoryMarshal.GetReference(TOperator.Taps); + Initialize(buffer, above, left, width, height, Unsafe.Subtract(ref aboveBase, 1)); + PredictGroupsScalar(buffer, width, height, (1 << bitDepth) - 1, ref taps); + CopyToDestinationScalar(buffer, destination, destinationStride, width, height); + } + + /// + /// Initializes an 8-bit recursive filter-intra workspace from the prepared top and left references. + /// + /// The recursive workspace. + /// The top reference. + /// The left reference. + /// The block width. + /// The block height. + /// The shared top-left reference. + private static void Initialize(Span buffer, ReadOnlySpan above, ReadOnlySpan left, int width, int height, byte topLeft) + { + buffer[0] = topLeft; + above[..width].CopyTo(buffer[1..]); + for (int row = 0; row < height; row++) + { + buffer[(row + 1) * BufferStride] = left[row]; + } + } + + /// + /// Initializes a high-bit-depth recursive filter-intra workspace from the prepared top and left references. + /// + /// The recursive workspace. + /// The top reference. + /// The left reference. + /// The block width. + /// The block height. + /// The shared top-left reference. + private static void Initialize(Span buffer, ReadOnlySpan above, ReadOnlySpan left, int width, int height, short topLeft) + { + buffer[0] = topLeft; + above[..width].CopyTo(buffer[1..]); + for (int row = 0; row < height; row++) + { + buffer[(row + 1) * BufferStride] = left[row]; + } + } + + /// + /// Creates the coefficients for eight output samples at one tap position. + /// + /// The first coefficient in the operator table. + /// The tap position. + /// The coefficient vector. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 CreateTapVector256(ref sbyte taps, int tap) + => Vector256.Create( + (int)Unsafe.Add(ref taps, tap), + Unsafe.Add(ref taps, 7 + tap), + Unsafe.Add(ref taps, 14 + tap), + Unsafe.Add(ref taps, 21 + tap), + Unsafe.Add(ref taps, 28 + tap), + Unsafe.Add(ref taps, 35 + tap), + Unsafe.Add(ref taps, 42 + tap), + Unsafe.Add(ref taps, 49 + tap)); + + /// + /// Creates the coefficients for four output samples at one tap position. + /// + /// The first coefficient in the operator table. + /// The tap position. + /// The first output sample. + /// The coefficient vector. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateTapVector128(ref sbyte taps, int tap, int pixel) + => Vector128.Create( + (int)Unsafe.Add(ref taps, (pixel * 7) + tap), + Unsafe.Add(ref taps, ((pixel + 1) * 7) + tap), + Unsafe.Add(ref taps, ((pixel + 2) * 7) + tap), + Unsafe.Add(ref taps, ((pixel + 3) * 7) + tap)); + + /// + /// Calculates eight filtered predictions from seven reconstructed samples. + /// + /// The first tap coefficients. + /// The second tap coefficients. + /// The third tap coefficients. + /// The fourth tap coefficients. + /// The fifth tap coefficients. + /// The sixth tap coefficients. + /// The seventh tap coefficients. + /// The first reconstructed sample. + /// The second reconstructed sample. + /// The third reconstructed sample. + /// The fourth reconstructed sample. + /// The fifth reconstructed sample. + /// The sixth reconstructed sample. + /// The seventh reconstructed sample. + /// The unrounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Calculate( + Vector256 tap0, + Vector256 tap1, + Vector256 tap2, + Vector256 tap3, + Vector256 tap4, + Vector256 tap5, + Vector256 tap6, + int p0, + int p1, + int p2, + int p3, + int p4, + int p5, + int p6) + => (tap0 * p0) + (tap1 * p1) + (tap2 * p2) + (tap3 * p3) + (tap4 * p4) + (tap5 * p5) + (tap6 * p6); + + /// + /// Calculates four filtered predictions from seven reconstructed samples. + /// + /// The first tap coefficients. + /// The second tap coefficients. + /// The third tap coefficients. + /// The fourth tap coefficients. + /// The fifth tap coefficients. + /// The sixth tap coefficients. + /// The seventh tap coefficients. + /// The first reconstructed sample. + /// The second reconstructed sample. + /// The third reconstructed sample. + /// The fourth reconstructed sample. + /// The fifth reconstructed sample. + /// The sixth reconstructed sample. + /// The seventh reconstructed sample. + /// The unrounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Calculate( + Vector128 tap0, + Vector128 tap1, + Vector128 tap2, + Vector128 tap3, + Vector128 tap4, + Vector128 tap5, + Vector128 tap6, + int p0, + int p1, + int p2, + int p3, + int p4, + int p5, + int p6) + => (tap0 * p0) + (tap1 * p1) + (tap2 * p2) + (tap3 * p3) + (tap4 * p4) + (tap5 * p5) + (tap6 * p6); + + /// + /// Stores eight 8-bit predictions into the two recursive output rows. + /// + /// The clamped predictions. + /// The recursive workspace origin. + /// The first output-row offset. + /// The second output-row offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreEightBytes(Vector256 prediction, ref byte buffer, int firstRowOffset, int secondRowOffset) + { + Vector256 narrowed16 = Vector256.Narrow(prediction.AsUInt32(), Vector256.Zero); + Vector256 narrowed8 = Vector256.Narrow(narrowed16, Vector256.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, firstRowOffset)) = narrowed8.AsUInt32().GetElement(0); + Unsafe.As(ref Unsafe.Add(ref buffer, secondRowOffset)) = narrowed8.AsUInt32().GetElement(1); + } + + /// + /// Stores four 8-bit predictions into one recursive output row. + /// + /// The clamped predictions. + /// The recursive workspace origin. + /// The output-row offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreFourBytes(Vector128 prediction, ref byte buffer, int offset) + { + Vector128 narrowed16 = Vector128.Narrow(prediction.AsUInt32(), Vector128.Zero); + Vector128 narrowed8 = Vector128.Narrow(narrowed16, Vector128.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, offset)) = narrowed8.AsUInt32().GetElement(0); + } + + /// + /// Stores eight high-bit-depth predictions into the two recursive output rows. + /// + /// The clamped predictions. + /// The recursive workspace origin. + /// The first output-row offset. + /// The second output-row offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreEightShorts(Vector256 prediction, ref short buffer, int firstRowOffset, int secondRowOffset) + { + Vector256 narrowed = Vector256.Narrow(prediction, Vector256.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, firstRowOffset)) = narrowed.AsUInt64().GetElement(0); + Unsafe.As(ref Unsafe.Add(ref buffer, secondRowOffset)) = narrowed.AsUInt64().GetElement(1); + } + + /// + /// Stores four high-bit-depth predictions into one recursive output row. + /// + /// The clamped predictions. + /// The recursive workspace origin. + /// The output-row offset. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreFourShorts(Vector128 prediction, ref short buffer, int offset) + { + Vector128 narrowed = Vector128.Narrow(prediction, Vector128.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, offset)) = narrowed.AsUInt64().GetElement(0); + } + + /// + /// Calculates every 8-bit recursive filter group without hardware intrinsics. + /// + /// The initialized recursive workspace. + /// The block width. + /// The block height. + /// The maximum reconstructed sample. + /// The first filter coefficient. + private static void PredictGroupsScalar(Span buffer, int width, int height, int maximum, ref sbyte taps) + { + ref byte bufferBase = ref MemoryMarshal.GetReference(buffer); + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + int p0 = Unsafe.Add(ref bufferBase, sourceOffset); + int p1 = Unsafe.Add(ref bufferBase, sourceOffset + 1); + int p2 = Unsafe.Add(ref bufferBase, sourceOffset + 2); + int p3 = Unsafe.Add(ref bufferBase, sourceOffset + 3); + int p4 = Unsafe.Add(ref bufferBase, sourceOffset + 4); + int p5 = Unsafe.Add(ref bufferBase, sourceOffset + BufferStride); + int p6 = Unsafe.Add(ref bufferBase, sourceOffset + (2 * BufferStride)); + for (int pixel = 0; pixel < 8; pixel++) + { + int tapOffset = pixel * 7; + int prediction = + (Unsafe.Add(ref taps, tapOffset) * p0) + + (Unsafe.Add(ref taps, tapOffset + 1) * p1) + + (Unsafe.Add(ref taps, tapOffset + 2) * p2) + + (Unsafe.Add(ref taps, tapOffset + 3) * p3) + + (Unsafe.Add(ref taps, tapOffset + 4) * p4) + + (Unsafe.Add(ref taps, tapOffset + 5) * p5) + + (Unsafe.Add(ref taps, tapOffset + 6) * p6); + + int destinationOffset = ((row + (pixel >> 2)) * BufferStride) + column + (pixel & 3); + Unsafe.Add(ref bufferBase, destinationOffset) = (byte)Math.Clamp((prediction + 8) >> 4, 0, maximum); + } + } + } + } + + /// + /// Calculates every high-bit-depth recursive filter group without hardware intrinsics. + /// + /// The initialized recursive workspace. + /// The block width. + /// The block height. + /// The maximum reconstructed sample. + /// The first filter coefficient. + private static void PredictGroupsScalar(Span buffer, int width, int height, int maximum, ref sbyte taps) + { + ref short bufferBase = ref MemoryMarshal.GetReference(buffer); + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + int p0 = Unsafe.Add(ref bufferBase, sourceOffset); + int p1 = Unsafe.Add(ref bufferBase, sourceOffset + 1); + int p2 = Unsafe.Add(ref bufferBase, sourceOffset + 2); + int p3 = Unsafe.Add(ref bufferBase, sourceOffset + 3); + int p4 = Unsafe.Add(ref bufferBase, sourceOffset + 4); + int p5 = Unsafe.Add(ref bufferBase, sourceOffset + BufferStride); + int p6 = Unsafe.Add(ref bufferBase, sourceOffset + (2 * BufferStride)); + for (int pixel = 0; pixel < 8; pixel++) + { + int tapOffset = pixel * 7; + int prediction = + (Unsafe.Add(ref taps, tapOffset) * p0) + + (Unsafe.Add(ref taps, tapOffset + 1) * p1) + + (Unsafe.Add(ref taps, tapOffset + 2) * p2) + + (Unsafe.Add(ref taps, tapOffset + 3) * p3) + + (Unsafe.Add(ref taps, tapOffset + 4) * p4) + + (Unsafe.Add(ref taps, tapOffset + 5) * p5) + + (Unsafe.Add(ref taps, tapOffset + 6) * p6); + + int destinationOffset = ((row + (pixel >> 2)) * BufferStride) + column + (pixel & 3); + Unsafe.Add(ref bufferBase, destinationOffset) = (short)Math.Clamp((prediction + 8) >> 4, 0, maximum); + } + } + } + } + + /// + /// Copies an 8-bit recursive workspace into the strided destination. + /// + /// The completed recursive workspace. + /// The destination block origin. + /// The destination row stride. + /// The block width. + /// The block height. + private static void CopyToDestination(ReadOnlySpan buffer, Span destination, int destinationStride, int width, int height) + { + for (int row = 0; row < height; row++) + { + buffer.Slice(((row + 1) * BufferStride) + 1, width).CopyTo(destination.Slice(row * destinationStride, width)); + } + } + + /// + /// Copies a high-bit-depth recursive workspace into the strided destination. + /// + /// The completed recursive workspace. + /// The destination block origin. + /// The destination row stride. + /// The block width. + /// The block height. + private static void CopyToDestination(ReadOnlySpan buffer, Span destination, int destinationStride, int width, int height) + { + for (int row = 0; row < height; row++) + { + buffer.Slice(((row + 1) * BufferStride) + 1, width).CopyTo(destination.Slice(row * destinationStride, width)); + } + } + + /// + /// Copies an 8-bit recursive workspace into the strided destination without vectorized span copying. + /// + /// The completed recursive workspace. + /// The destination block origin. + /// The destination row stride. + /// The block width. + /// The block height. + private static void CopyToDestinationScalar(ReadOnlySpan buffer, Span destination, int destinationStride, int width, int height) + { + for (int row = 0; row < height; row++) + { + int sourceOffset = ((row + 1) * BufferStride) + 1; + int destinationOffset = row * destinationStride; + for (int column = 0; column < width; column++) + { + destination[destinationOffset + column] = buffer[sourceOffset + column]; + } + } + } + + /// + /// Copies a high-bit-depth recursive workspace into the strided destination without vectorized span copying. + /// + /// The completed recursive workspace. + /// The destination block origin. + /// The destination row stride. + /// The block width. + /// The block height. + private static void CopyToDestinationScalar(ReadOnlySpan buffer, Span destination, int destinationStride, int width, int height) + { + for (int row = 0; row < height; row++) + { + int sourceOffset = ((row + 1) * BufferStride) + 1; + int destinationOffset = row * destinationStride; + for (int column = 0; column < width; column++) + { + destination[destinationOffset + column] = buffer[sourceOffset + column]; + } + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.OperatorContract.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.OperatorContract.cs new file mode 100644 index 0000000000..5e698f85b8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.OperatorContract.cs @@ -0,0 +1,25 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Defines the coefficient set for one AV1 filter-intra prediction mode. + /// + internal interface IAv1FilterIntraPredictionOperator + { + /// + /// Gets the filter-intra mode implemented by the operator. + /// + public static abstract Av1FilterIntraMode Mode { get; } + + /// + /// Gets the eight seven-tap coefficient rows used by the operator. + /// + public static abstract ReadOnlySpan Taps { get; } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.PaethOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.PaethOperator.cs new file mode 100644 index 0000000000..bbbc93ab3b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.PaethOperator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Defines the AV1 Paeth filter-intra coefficient operator. + /// + internal readonly struct PaethOperator : IAv1FilterIntraPredictionOperator + { + /// + public static Av1FilterIntraMode Mode => Av1FilterIntraMode.Paeth; + + /// + public static ReadOnlySpan Taps => + [ + -12, 14, 0, 0, 0, 14, 0, + -10, 0, 14, 0, 0, 12, 0, + -9, 0, 0, 14, 0, 11, 0, + -8, 0, 0, 0, 14, 10, 0, + -10, 12, 0, 0, 0, 0, 14, + -9, 1, 12, 0, 0, 0, 12, + -8, 0, 0, 12, 0, 1, 11, + -7, 0, 0, 1, 12, 1, 9, + ]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.VerticalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.VerticalOperator.cs new file mode 100644 index 0000000000..401621a015 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.VerticalOperator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// Defines the AV1 vertical filter-intra coefficient operator. + /// + internal readonly struct VerticalOperator : IAv1FilterIntraPredictionOperator + { + /// + public static Av1FilterIntraMode Mode => Av1FilterIntraMode.Vertical; + + /// + public static ReadOnlySpan Taps => + [ + -10, 16, 0, 0, 0, 10, 0, + -6, 0, 16, 0, 0, 6, 0, + -4, 0, 0, 16, 0, 4, 0, + -2, 0, 0, 0, 16, 2, 0, + -10, 16, 0, 0, 0, 0, 10, + -6, 0, 16, 0, 0, 0, 6, + -4, 0, 0, 16, 0, 0, 4, + -2, 0, 0, 0, 16, 0, 2, + ]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs deleted file mode 100644 index 0f2217fa8b..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Produces AV1 filter intra predictions from reconstructed neighboring samples. -/// -/// -/// The scalar prediction follows the filter intra prediction process in section 7.11.2.3 of the AV1 specification. -/// -internal static class Av1FilterIntraPredictor -{ - /// - /// The row stride of the temporary prediction buffer. - /// - private const int BufferStride = 33; - - /// - /// The number of samples in the temporary prediction buffer. - /// - private const int BufferLength = BufferStride * BufferStride; - - /// - /// The number of nonzero filter coefficients used to predict each sample. - /// - private const int TapsPerPixel = 7; - - /// - /// The number of samples produced by each filter coefficient group. - /// - private const int PixelsPerGroup = 8; - - /// - /// The number of stored coefficients for each filter intra mode. - /// - private const int TapsPerMode = TapsPerPixel * PixelsPerGroup; - - /// - /// Gets the filter coefficients for the five AV1 filter intra modes. - /// - /// - /// AV1 defines eight filters per mode over seven nonzero neighboring samples. The eighth coefficient used by - /// libaom is always zero, so it is omitted here to keep the scalar coefficient layout aligned with the work performed. - /// - public static readonly sbyte[] Taps = - [ - - // DC - -6, 10, 0, 0, 0, 12, 0, - -5, 2, 10, 0, 0, 9, 0, - -3, 1, 1, 10, 0, 7, 0, - -3, 1, 1, 2, 10, 5, 0, - -4, 6, 0, 0, 0, 2, 12, - -3, 2, 6, 0, 0, 2, 9, - -3, 2, 2, 6, 0, 2, 7, - -3, 1, 2, 2, 6, 3, 5, - - // Vertical - -10, 16, 0, 0, 0, 10, 0, - -6, 0, 16, 0, 0, 6, 0, - -4, 0, 0, 16, 0, 4, 0, - -2, 0, 0, 0, 16, 2, 0, - -10, 16, 0, 0, 0, 0, 10, - -6, 0, 16, 0, 0, 0, 6, - -4, 0, 0, 16, 0, 0, 4, - -2, 0, 0, 0, 16, 0, 2, - - // Horizontal - -8, 8, 0, 0, 0, 16, 0, - -8, 0, 8, 0, 0, 16, 0, - -8, 0, 0, 8, 0, 16, 0, - -8, 0, 0, 0, 8, 16, 0, - -4, 4, 0, 0, 0, 0, 16, - -4, 0, 4, 0, 0, 0, 16, - -4, 0, 0, 4, 0, 0, 16, - -4, 0, 0, 0, 4, 0, 16, - - // Directional 157 degrees - -2, 8, 0, 0, 0, 10, 0, - -1, 3, 8, 0, 0, 6, 0, - -1, 2, 3, 8, 0, 4, 0, - 0, 1, 2, 3, 8, 2, 0, - -1, 4, 0, 0, 0, 3, 10, - -1, 3, 4, 0, 0, 4, 6, - -1, 2, 3, 4, 0, 4, 4, - -1, 2, 2, 3, 4, 3, 3, - - // Paeth - -12, 14, 0, 0, 0, 14, 0, - -10, 0, 14, 0, 0, 12, 0, - -9, 0, 0, 14, 0, 11, 0, - -8, 0, 0, 0, 14, 10, 0, - -10, 12, 0, 0, 0, 0, 14, - -9, 1, 12, 0, 0, 0, 12, - -8, 0, 0, 12, 0, 1, 11, - -7, 0, 0, 1, 12, 1, 9, - ]; - - /// - /// Produces an 8-bit filter intra prediction for a transform block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The transform size that determines the prediction block dimensions. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The filter intra mode whose coefficient set is applied. - public static void Predict( - Span destination, - nuint destinationStride, - Av1TransformSize transformSize, - Span above, - Span left, - Av1FilterIntraMode mode) - { - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - DebugGuard.MustBeLessThanOrEqualTo(width, 32, nameof(width)); - DebugGuard.MustBeLessThanOrEqualTo(height, 32, nameof(height)); - Guard.MustBeGreaterThanOrEqualTo(destinationStride, (nuint)width, nameof(destinationStride)); - Guard.MustBeSizedAtLeast(destination, (int)destinationStride * height, nameof(destination)); - Guard.MustBeSizedAtLeast(above, width, nameof(above)); - Guard.MustBeSizedAtLeast(left, height, nameof(left)); - - Span buffer = stackalloc byte[BufferLength]; - ref byte bufferRef = ref buffer[0]; - ref byte aboveRef = ref above[0]; - ref byte leftRef = ref left[0]; - - // Row zero includes the top-left sample followed by the top neighbors. - // Column zero stores the left neighbors so each 4-by-2 group can consume the - // seven already-reconstructed samples defined by the recursive AV1 process. - bufferRef = Unsafe.Subtract(ref aboveRef, 1); - above[..width].CopyTo(buffer[1..]); - for (int row = 0; row < height; row++) - { - Unsafe.Add(ref bufferRef, (row + 1) * BufferStride) = Unsafe.Add(ref leftRef, row); - } - - int modeOffset = (int)mode * TapsPerMode; - ref sbyte tapsRef = ref Taps[modeOffset]; - for (int row = 1; row <= height; row += 2) - { - for (int column = 1; column <= width; column += 4) - { - int sourceOffset = ((row - 1) * BufferStride) + column - 1; - int p0 = Unsafe.Add(ref bufferRef, sourceOffset); - int p1 = Unsafe.Add(ref bufferRef, sourceOffset + 1); - int p2 = Unsafe.Add(ref bufferRef, sourceOffset + 2); - int p3 = Unsafe.Add(ref bufferRef, sourceOffset + 3); - int p4 = Unsafe.Add(ref bufferRef, sourceOffset + 4); - int p5 = Unsafe.Add(ref bufferRef, sourceOffset + BufferStride); - int p6 = Unsafe.Add(ref bufferRef, sourceOffset + (2 * BufferStride)); - - for (int pixel = 0; pixel < PixelsPerGroup; pixel++) - { - int tapOffset = pixel * TapsPerPixel; - int prediction = - (Unsafe.Add(ref tapsRef, tapOffset) * p0) - + (Unsafe.Add(ref tapsRef, tapOffset + 1) * p1) - + (Unsafe.Add(ref tapsRef, tapOffset + 2) * p2) - + (Unsafe.Add(ref tapsRef, tapOffset + 3) * p3) - + (Unsafe.Add(ref tapsRef, tapOffset + 4) * p4) - + (Unsafe.Add(ref tapsRef, tapOffset + 5) * p5) - + (Unsafe.Add(ref tapsRef, tapOffset + 6) * p6); - - int rowOffset = pixel >> 2; - int columnOffset = pixel & 3; - int destinationOffset = ((row + rowOffset) * BufferStride) + column + columnOffset; - Unsafe.Add(ref bufferRef, destinationOffset) = - (byte)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 4), 0, 255); - } - } - } - - ref byte destinationRef = ref destination[0]; - for (int row = 0; row < height; row++) - { - buffer.Slice(((row + 1) * BufferStride) + 1, width).CopyTo( - MemoryMarshal.CreateSpan(ref destinationRef, width)); - - destinationRef = ref Unsafe.Add(ref destinationRef, destinationStride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictorBase.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictorBase.cs new file mode 100644 index 0000000000..7d59507115 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictorBase.cs @@ -0,0 +1,121 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Reconstructs AV1 filter-intra prediction blocks from prepared neighboring samples. +/// +/// +/// The implementation follows the recursive filter-intra process in section 7.11.2.3 of the AV1 specification. +/// +internal abstract partial class Av1FilterIntraPredictorBase +{ + /// + /// The row stride of the recursive prediction workspace. + /// + public const int BufferStride = 33; + + /// + /// The number of samples in the recursive prediction workspace. + /// + public const int ScratchLength = BufferStride * BufferStride; + + /// + /// The DC filter-intra predictor. + /// + private static readonly Av1FilterIntraPredictor DcPredictor = new(); + + /// + /// The vertical filter-intra predictor. + /// + private static readonly Av1FilterIntraPredictor VerticalPredictor = new(); + + /// + /// The horizontal filter-intra predictor. + /// + private static readonly Av1FilterIntraPredictor HorizontalPredictor = new(); + + /// + /// The 157-degree directional filter-intra predictor. + /// + private static readonly Av1FilterIntraPredictor Directional157Predictor = new(); + + /// + /// The Paeth filter-intra predictor. + /// + private static readonly Av1FilterIntraPredictor PaethPredictor = new(); + + /// + /// Gets the filter-intra mode implemented by this predictor. + /// + public abstract Av1FilterIntraMode Mode { get; } + + /// + /// Gets the closed predictor for a filter-intra mode. + /// + /// The decoded filter-intra mode. + /// The predictor for . + public static Av1FilterIntraPredictorBase GetPredictor(Av1FilterIntraMode mode) + => mode switch + { + Av1FilterIntraMode.DC => DcPredictor, + Av1FilterIntraMode.Vertical => VerticalPredictor, + Av1FilterIntraMode.Horizontal => HorizontalPredictor, + Av1FilterIntraMode.Directional157 => Directional157Predictor, + Av1FilterIntraMode.Paeth => PaethPredictor, + _ => throw new InvalidImageContentException($"Filter-intra mode {mode} is not defined by AV1."), + }; + + /// + /// Predicts an 8-bit filter-intra block. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width. + /// The block height. + /// The caller-owned recursive prediction workspace. + public abstract void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, Span scratch); + + /// + /// Predicts a high-bit-depth filter-intra block. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width. + /// The block height. + /// The reconstructed sample precision. + /// The caller-owned recursive prediction workspace. + public abstract void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, int bitDepth, Span scratch); + + /// + /// Predicts an 8-bit filter-intra block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width. + /// The block height. + /// The caller-owned recursive prediction workspace. + public abstract void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, Span scratch); + + /// + /// Predicts a high-bit-depth filter-intra block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width. + /// The block height. + /// The reconstructed sample precision. + /// The caller-owned recursive prediction workspace. + public abstract void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height, int bitDepth, Span scratch); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1HighBitDepthPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1HighBitDepthPredictor.cs deleted file mode 100644 index 57b59f6a8b..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1HighBitDepthPredictor.cs +++ /dev/null @@ -1,602 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Implements AV1 intra prediction for 10-bit and 12-bit sample buffers. -/// -/// -/// Samples are stored in signed 16-bit buffers, but predictions are clamped to the nonnegative range of the signaled bit depth. -/// -internal static class Av1HighBitDepthPredictor -{ - /// - /// The row stride of the temporary filter intra prediction buffer. - /// - private const int FilterBufferStride = 33; - - /// - /// The number of samples in the temporary filter intra prediction buffer. - /// - private const int FilterBufferLength = FilterBufferStride * FilterBufferStride; - - /// - /// The number of nonzero filter coefficients used to predict each filter intra sample. - /// - private const int FilterTapsPerPixel = 7; - - /// - /// The number of samples produced by each filter coefficient group. - /// - private const int FilterPixelsPerGroup = 8; - - /// - /// The number of stored coefficients for each filter intra mode. - /// - private const int FilterTapsPerMode = FilterTapsPerPixel * FilterPixelsPerGroup; - - /// - /// Produces a high-bit-depth DC intra prediction from the available neighboring samples. - /// - /// A value indicating whether reconstructed left samples are available. - /// A value indicating whether reconstructed top samples are available. - /// The transform size that determines the prediction block dimensions. - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The number of bits used to represent each sample. - public static void DcPredictor( - bool hasLeft, - bool hasAbove, - Av1TransformSize transformSize, - Span destination, - nuint destinationStride, - Span above, - Span left, - int bitDepth) - { - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - int sum = 0; - int count = 0; - - if (hasAbove) - { - for (int column = 0; column < width; column++) - { - sum += above[column]; - } - - count += width; - } - - if (hasLeft) - { - for (int row = 0; row < height; row++) - { - sum += left[row]; - } - - count += height; - } - - short prediction = count == 0 - ? (short)(1 << (bitDepth - 1)) - : (short)((sum + (count >> 1)) / count); - - // The midpoint is normative when neither edge exists; otherwise the half-count bias - // rounds the mean of the available top and left samples to the nearest integer. - Fill(destination, destinationStride, width, height, prediction); - } - - /// - /// Produces a high-bit-depth nondirectional intra prediction. - /// - /// The nondirectional prediction mode to apply. - /// The transform size that determines the prediction block dimensions. - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - public static void GeneralPredictor( - Av1PredictionMode mode, - Av1TransformSize transformSize, - Span destination, - nuint destinationStride, - Span above, - Span left) - { - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - - switch (mode) - { - case Av1PredictionMode.Horizontal: - PredictHorizontal(destination, destinationStride, left, width, height); - break; - case Av1PredictionMode.Vertical: - PredictVertical(destination, destinationStride, above, width, height); - break; - case Av1PredictionMode.Paeth: - PredictPaeth(destination, destinationStride, above, left, width, height); - break; - case Av1PredictionMode.Smooth: - PredictSmooth(destination, destinationStride, above, left, width, height); - break; - case Av1PredictionMode.SmoothHorizontal: - PredictSmoothHorizontal(destination, destinationStride, above, left, width, height); - break; - case Av1PredictionMode.SmoothVertical: - PredictSmoothVertical(destination, destinationStride, above, left, width, height); - break; - } - } - - /// - /// Produces a high-bit-depth directional intra prediction. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The transform size that determines the prediction block dimensions. - /// The top reference samples, including any required extension. - /// The left reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// A value indicating whether the left reference samples were upsampled. - /// The prediction angle in degrees. - /// The number of bits used to represent each sample. - public static void DirectionalPredictor( - Span destination, - nuint destinationStride, - Av1TransformSize transformSize, - Span above, - Span left, - bool upsampleAbove, - bool upsampleLeft, - int angle, - int bitDepth) - { - Guard.MustBeBetweenOrEqualTo(angle, 1, 269, nameof(angle)); - int dx = Av1PredictorFactory.GetDeltaX(angle); - int dy = Av1PredictorFactory.GetDeltaY(angle); - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - - // Angles on a cardinal axis copy one reference edge directly. Other angles are - // separated into the three AV1 projection zones according to the edges they cross. - if (angle is > 0 and < 90) - { - PredictDirectionalZone1(destination, destinationStride, above, upsampleAbove, dx, width, height, bitDepth); - } - else if (angle is > 90 and < 180) - { - PredictDirectionalZone2(destination, destinationStride, above, left, upsampleAbove, upsampleLeft, dx, dy, width, height, bitDepth); - } - else if (angle is > 180 and < 270) - { - PredictDirectionalZone3(destination, destinationStride, left, upsampleLeft, dx, dy, width, height, bitDepth); - } - else if (angle == 90) - { - PredictVertical(destination, destinationStride, above, width, height); - } - else if (angle == 180) - { - PredictHorizontal(destination, destinationStride, left, width, height); - } - } - - /// - /// Produces a high-bit-depth filter intra prediction. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The transform size that determines the prediction block dimensions. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The filter intra mode whose coefficient set is applied. - /// The number of bits used to represent each sample. - public static void FilterIntraPredictor( - Span destination, - nuint destinationStride, - Av1TransformSize transformSize, - Span above, - Span left, - Av1FilterIntraMode mode, - int bitDepth) - { - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - int maximum = (1 << bitDepth) - 1; - DebugGuard.MustBeLessThanOrEqualTo(width, 32, nameof(width)); - DebugGuard.MustBeLessThanOrEqualTo(height, 32, nameof(height)); - Guard.MustBeGreaterThanOrEqualTo(destinationStride, (nuint)width, nameof(destinationStride)); - Guard.MustBeSizedAtLeast(destination, (int)destinationStride * height, nameof(destination)); - Guard.MustBeSizedAtLeast(above, width, nameof(above)); - Guard.MustBeSizedAtLeast(left, height, nameof(left)); - - Span buffer = stackalloc short[FilterBufferLength]; - ref short bufferRef = ref buffer[0]; - ref short aboveRef = ref above[0]; - ref short leftRef = ref left[0]; - - // Row zero includes the top-left sample followed by the top neighbors. - // Column zero stores the left neighbors so each 4-by-2 group can consume the - // seven already-reconstructed samples defined by the recursive AV1 process. - bufferRef = Unsafe.Subtract(ref aboveRef, 1); - above[..width].CopyTo(buffer[1..]); - for (int row = 0; row < height; row++) - { - Unsafe.Add(ref bufferRef, (row + 1) * FilterBufferStride) = Unsafe.Add(ref leftRef, row); - } - - int modeOffset = (int)mode * FilterTapsPerMode; - ref sbyte tapsRef = ref Av1FilterIntraPredictor.Taps[modeOffset]; - for (int row = 1; row <= height; row += 2) - { - for (int column = 1; column <= width; column += 4) - { - int sourceOffset = ((row - 1) * FilterBufferStride) + column - 1; - int p0 = Unsafe.Add(ref bufferRef, sourceOffset); - int p1 = Unsafe.Add(ref bufferRef, sourceOffset + 1); - int p2 = Unsafe.Add(ref bufferRef, sourceOffset + 2); - int p3 = Unsafe.Add(ref bufferRef, sourceOffset + 3); - int p4 = Unsafe.Add(ref bufferRef, sourceOffset + 4); - int p5 = Unsafe.Add(ref bufferRef, sourceOffset + FilterBufferStride); - int p6 = Unsafe.Add(ref bufferRef, sourceOffset + (2 * FilterBufferStride)); - - for (int pixel = 0; pixel < FilterPixelsPerGroup; pixel++) - { - int tapOffset = pixel * FilterTapsPerPixel; - int prediction = - (Unsafe.Add(ref tapsRef, tapOffset) * p0) - + (Unsafe.Add(ref tapsRef, tapOffset + 1) * p1) - + (Unsafe.Add(ref tapsRef, tapOffset + 2) * p2) - + (Unsafe.Add(ref tapsRef, tapOffset + 3) * p3) - + (Unsafe.Add(ref tapsRef, tapOffset + 4) * p4) - + (Unsafe.Add(ref tapsRef, tapOffset + 5) * p5) - + (Unsafe.Add(ref tapsRef, tapOffset + 6) * p6); - - int rowOffset = pixel >> 2; - int columnOffset = pixel & 3; - int destinationOffset = ((row + rowOffset) * FilterBufferStride) + column + columnOffset; - Unsafe.Add(ref bufferRef, destinationOffset) = - (short)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 4), 0, maximum); - } - } - } - - for (int row = 0; row < height; row++) - { - buffer.Slice(((row + 1) * FilterBufferStride) + 1, width).CopyTo( - destination.Slice(row * (int)destinationStride, width)); - } - } - - /// - /// Copies each left reference sample across one destination row. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed left reference samples. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - private static void PredictHorizontal(Span destination, nuint stride, Span left, int width, int height) - { - for (int row = 0; row < height; row++) - { - destination.Slice(row * (int)stride, width).Fill(left[row]); - } - } - - /// - /// Copies the top reference samples into every destination row. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - private static void PredictVertical(Span destination, nuint stride, Span above, int width, int height) - { - for (int row = 0; row < height; row++) - { - above[..width].CopyTo(destination.Slice(row * (int)stride, width)); - } - } - - /// - /// Produces a Paeth prediction from the nearest top, left, and top-left reference sample. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - private static void PredictPaeth(Span destination, nuint stride, Span above, Span left, int width, int height) - { - int topLeft = Unsafe.Subtract(ref above[0], 1); - for (int row = 0; row < height; row++) - { - Span destinationRow = destination.Slice(row * (int)stride, width); - for (int column = 0; column < width; column++) - { - short leftValue = left[row]; - short topValue = above[column]; - int basis = topValue + leftValue - topLeft; - int leftDistance = Av1Math.AbsoluteDifference(basis, leftValue); - int topDistance = Av1Math.AbsoluteDifference(basis, topValue); - int topLeftDistance = Av1Math.AbsoluteDifference(basis, topLeft); - - // The comparison order preserves AV1's left, top, then top-left tie precedence. - destinationRow[column] = leftDistance <= topDistance && leftDistance <= topLeftDistance - ? leftValue - : topDistance <= topLeftDistance ? topValue : (short)topLeft; - } - } - } - - /// - /// Produces a two-dimensional smooth prediction from the four terminating edge samples. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - private static void PredictSmooth(Span destination, nuint stride, Span above, Span left, int width, int height) - { - int below = left[height - 1]; - int right = above[width - 1]; - Span widthWeights = Av1SmoothPredictor.Weights.AsSpan(width, width); - Span heightWeights = Av1SmoothPredictor.Weights.AsSpan(height, height); - int scale = 1 << Av1SmoothPredictor.WeightLog2Scale; - int log2Scale = Av1SmoothPredictor.WeightLog2Scale + 1; - - // Horizontal weights follow the block width and vertical weights follow the height. - // Keeping those domains separate is required for rectangular transform blocks. - for (int row = 0; row < height; row++) - { - int rowWeight = heightWeights[row]; - Span destinationRow = destination.Slice(row * (int)stride, width); - for (int column = 0; column < width; column++) - { - int columnWeight = widthWeights[column]; - int prediction = (above[column] * rowWeight) + (below * (scale - rowWeight)); - prediction += (left[row] * columnWeight) + (right * (scale - columnWeight)); - destinationRow[column] = (short)Av1Math.DivideRound(prediction, log2Scale); - } - } - } - - /// - /// Produces a horizontal smooth prediction between the left and right edge samples. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - private static void PredictSmoothHorizontal(Span destination, nuint stride, Span above, Span left, int width, int height) - { - int right = above[width - 1]; - Span weights = Av1SmoothPredictor.Weights.AsSpan(width, width); - int scale = 1 << Av1SmoothPredictor.WeightLog2Scale; - - for (int row = 0; row < height; row++) - { - Span destinationRow = destination.Slice(row * (int)stride, width); - for (int column = 0; column < width; column++) - { - int weight = weights[column]; - int prediction = (left[row] * weight) + (right * (scale - weight)); - destinationRow[column] = (short)Av1Math.DivideRound(prediction, Av1SmoothPredictor.WeightLog2Scale); - } - } - } - - /// - /// Produces a vertical smooth prediction between the top and bottom edge samples. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The reconstructed top reference samples. - /// The reconstructed left reference samples. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - private static void PredictSmoothVertical(Span destination, nuint stride, Span above, Span left, int width, int height) - { - int below = left[height - 1]; - Span weights = Av1SmoothPredictor.Weights.AsSpan(height, height); - int scale = 1 << Av1SmoothPredictor.WeightLog2Scale; - - for (int row = 0; row < height; row++) - { - int weight = weights[row]; - Span destinationRow = destination.Slice(row * (int)stride, width); - for (int column = 0; column < width; column++) - { - int prediction = (above[column] * weight) + (below * (scale - weight)); - destinationRow[column] = (short)Av1Math.DivideRound(prediction, Av1SmoothPredictor.WeightLog2Scale); - } - } - } - - /// - /// Projects top reference samples into a directional zone 1 prediction block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The top reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// The horizontal projection derivative in Q6 precision. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - /// The number of bits used to represent each sample. - private static void PredictDirectionalZone1(Span destination, nuint stride, Span above, bool upsample, int dx, int width, int height, int bitDepth) - { - int upsampleAbove = upsample ? 1 : 0; - int maxBasisX = (width + height - 1) << upsampleAbove; - int fractionBitCount = 6 - upsampleAbove; - int basisIncrement = 1 << upsampleAbove; - int maximum = (1 << bitDepth) - 1; - int x = dx; - ref short aboveRef = ref above[0]; - - for (int row = 0; row < height; row++) - { - Span destinationRow = destination.Slice(row * (int)stride, width); - - // AV1 retains six fractional projection bits, or five after reference upsampling. - // The interpolation weights sum to 32 because the low projection bit is discarded. - int basis = x >> fractionBitCount; - int shift = ((x << upsampleAbove) & 0x3F) >> 1; - for (int column = 0; column < width; column++) - { - if (basis < maxBasisX) - { - int prediction = (Unsafe.Add(ref aboveRef, basis) * (32 - shift)) + (Unsafe.Add(ref aboveRef, basis + 1) * shift); - destinationRow[column] = (short)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 5), 0, maximum); - } - else - { - destinationRow[column] = Unsafe.Add(ref aboveRef, maxBasisX); - } - - basis += basisIncrement; - } - - x += dx; - } - } - - /// - /// Projects top and left reference samples into a directional zone 2 prediction block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The top reference samples, including any required extension. - /// The left reference samples, including any required extension. - /// A value indicating whether the top reference samples were upsampled. - /// A value indicating whether the left reference samples were upsampled. - /// The horizontal projection derivative in Q6 precision. - /// The vertical projection derivative in Q6 precision. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - /// The number of bits used to represent each sample. - private static void PredictDirectionalZone2(Span destination, nuint stride, Span above, Span left, bool doUpsampleAbove, bool doUpsampleLeft, int dx, int dy, int width, int height, int bitDepth) - { - int upsampleAbove = doUpsampleAbove ? 1 : 0; - int upsampleLeft = doUpsampleLeft ? 1 : 0; - int minBasisX = -(1 << upsampleAbove); - int fractionBitCountX = 6 - upsampleAbove; - int fractionBitCountY = 6 - upsampleLeft; - int basisIncrementX = 1 << upsampleAbove; - int maximum = (1 << bitDepth) - 1; - int x = -dx; - ref short aboveRef = ref above[0]; - ref short leftRef = ref left[0]; - - for (int row = 0; row < height; row++) - { - Span destinationRow = destination.Slice(row * (int)stride, width); - int basisX = x >> fractionBitCountX; - int y = (row << 6) - dy; - for (int column = 0; column < width; column++, basisX += basisIncrementX, y -= dy) - { - int prediction; - - // A nonnegative top projection uses the above edge. Once the projection crosses - // the top-left corner, the same destination sample is projected from the left edge. - if (basisX >= minBasisX) - { - int shift = ((x * (1 << upsampleAbove)) & 0x3F) >> 1; - prediction = (Unsafe.Add(ref aboveRef, basisX) * (32 - shift)) + (Unsafe.Add(ref aboveRef, basisX + 1) * shift); - } - else - { - int basisY = y >> fractionBitCountY; - int shift = ((y * (1 << upsampleLeft)) & 0x3F) >> 1; - prediction = (Unsafe.Add(ref leftRef, basisY) * (32 - shift)) + (Unsafe.Add(ref leftRef, basisY + 1) * shift); - } - - destinationRow[column] = (short)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 5), 0, maximum); - } - - x -= dx; - } - } - - /// - /// Projects left reference samples into a directional zone 3 prediction block. - /// - /// The buffer that receives the predicted samples. - /// The distance, in samples, between destination rows. - /// The left reference samples, including any required extension. - /// A value indicating whether the left reference samples were upsampled. - /// The horizontal projection derivative, which must be one in zone 3. - /// The vertical projection derivative in Q6 precision. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - /// The number of bits used to represent each sample. - private static void PredictDirectionalZone3(Span destination, nuint stride, Span left, bool upsample, int dx, int dy, int width, int height, int bitDepth) - { - int upsampleLeft = upsample ? 1 : 0; - int maxBasisY = (width + height - 1) << upsampleLeft; - int fractionBitCount = 6 - upsampleLeft; - int basisIncrement = 1 << upsampleLeft; - int maximum = (1 << bitDepth) - 1; - int y = dy; - ref short leftRef = ref left[0]; - Guard.IsTrue(dx == 1, nameof(dx), "Dx expected to be always equal to 1 for directional Zone 3 prediction."); - - for (int column = 0; column < width; column++) - { - // Zone 3 is the transpose of zone 1: columns advance along the projected left edge, - // while rows advance through the reference samples for each destination column. - int basis = y >> fractionBitCount; - int shift = ((y << upsampleLeft) & 0x3F) >> 1; - for (int row = 0; row < height; row++) - { - int destinationOffset = (row * (int)stride) + column; - if (basis < maxBasisY) - { - int prediction = (Unsafe.Add(ref leftRef, basis) * (32 - shift)) + (Unsafe.Add(ref leftRef, basis + 1) * shift); - destination[destinationOffset] = (short)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 5), 0, maximum); - } - else - { - destination[destinationOffset] = Unsafe.Add(ref leftRef, maxBasisY); - } - - basis += basisIncrement; - } - - y += dy; - } - } - - /// - /// Fills a rectangular prediction block with one sample value. - /// - /// The buffer that receives the sample value. - /// The distance, in samples, between destination rows. - /// The width of the prediction block in samples. - /// The height of the prediction block in samples. - /// The sample value written to every destination position. - private static void Fill(Span destination, nuint stride, int width, int height, short value) - { - for (int row = 0; row < height; row++) - { - destination.Slice(row * (int)stride, width).Fill(value); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1HorizontalPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1HorizontalPredictor.cs deleted file mode 100644 index 3ae2e09b39..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1HorizontalPredictor.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block by extending each left neighboring sample across its destination row. -/// -internal readonly struct Av1HorizontalPredictor : IAv1Predictor -{ - /// - /// The number of samples written to each destination row. - /// - private readonly nuint blockWidth; - - /// - /// The number of left samples consumed and destination rows written. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1HorizontalPredictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1HorizontalPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block by extending its left edge horizontally. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The unused top-neighbor buffer required by the common predictor signature. - /// The left neighboring samples. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1HorizontalPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - /// SVT-AV1: highbd_h_predictor. - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte destinationRef = ref destination[0]; - uint width = (uint)this.blockWidth; - for (nuint r = 0; r < this.blockHeight; ++r) - { - Unsafe.InitBlock(ref destinationRef, Unsafe.Add(ref leftRef, r), width); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.HorizontalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.HorizontalOperator.cs new file mode 100644 index 0000000000..1781e88df0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.HorizontalOperator.cs @@ -0,0 +1,54 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Implements AV1 horizontal intra prediction for scalar and SIMD lanes. + /// + internal readonly struct HorizontalOperator : IAv1IntraPredictionOperator + { + /// + public static Av1PredictionMode Mode => Av1PredictionMode.Horizontal; + + /// + public static Av1IntraPredictionInputs Inputs => Av1IntraPredictionInputs.Left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) => left; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) => left; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs new file mode 100644 index 0000000000..cd4ca5eae4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs @@ -0,0 +1,344 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Applies one closed non-directional AV1 prediction operator using the widest available SIMD width. + /// + /// The prediction-mode-specific arithmetic. + internal sealed class Av1IntraPredictor : Av1IntraPredictorBase + where TOperator : struct, IAv1IntraPredictionOperator + { + /// + public override Av1PredictionMode Mode => TOperator.Mode; + + /// + public override void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height) + { + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref byte topBase = ref MemoryMarshal.GetReference(above); + ref byte leftBase = ref MemoryMarshal.GetReference(left); + ref int columnWeightBase = ref MemoryMarshal.GetReference(SmoothWeights[width..]); + + Av1IntraPredictionInputs inputs = TOperator.Inputs; + bool usesTop = (inputs & Av1IntraPredictionInputs.Top) != 0; + bool usesLeft = (inputs & Av1IntraPredictionInputs.Left) != 0; + bool usesTopLeft = (inputs & Av1IntraPredictionInputs.TopLeft) != 0; + bool usesTopRight = (inputs & Av1IntraPredictionInputs.TopRight) != 0; + bool usesBottomLeft = (inputs & Av1IntraPredictionInputs.BottomLeft) != 0; + bool usesColumnWeight = (inputs & Av1IntraPredictionInputs.ColumnWeight) != 0; + bool usesRowWeight = (inputs & Av1IntraPredictionInputs.RowWeight) != 0; + + byte topLeft = usesTopLeft ? Unsafe.Subtract(ref topBase, 1) : default; + byte topRight = usesTopRight ? Unsafe.Add(ref topBase, width - 1) : default; + byte bottomLeft = usesBottomLeft ? Unsafe.Add(ref leftBase, height - 1) : default; + int processedColumns = 0; + + if (Vector512.IsHardwareAccelerated) + { + int vectorizedColumns = width - (width % Vector512.Count); + if (vectorizedColumns > 0) + { + Vector512 topLeftVector = usesTopLeft ? Vector512.Create(topLeft) : default; + Vector512 topRightVector = usesTopRight ? Vector512.Create(topRight) : default; + Vector512 bottomLeftVector = usesBottomLeft ? Vector512.Create(bottomLeft) : default; + + for (int row = 0; row < height; row++) + { + Vector512 leftVector = usesLeft ? Vector512.Create(Unsafe.Add(ref leftBase, row)) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < vectorizedColumns; column += Vector512.Count) + { + Vector512 top = usesTop ? Vector512.LoadUnsafe(ref topBase, (nuint)column) : default; + ref int columnWeights = ref usesColumnWeight ? ref Unsafe.Add(ref columnWeightBase, column) : ref columnWeightBase; + Vector512 prediction = TOperator.Predict(top, leftVector, topLeftVector, topRightVector, bottomLeftVector, ref columnWeights, rowWeight); + prediction.StoreUnsafe(ref destinationRow, (nuint)column); + } + } + + processedColumns = vectorizedColumns; + } + } + + if (Vector256.IsHardwareAccelerated) + { + int remainingColumns = width - processedColumns; + int vectorizedColumns = remainingColumns - (remainingColumns % Vector256.Count); + if (vectorizedColumns > 0) + { + Vector256 topLeftVector = usesTopLeft ? Vector256.Create(topLeft) : default; + Vector256 topRightVector = usesTopRight ? Vector256.Create(topRight) : default; + Vector256 bottomLeftVector = usesBottomLeft ? Vector256.Create(bottomLeft) : default; + int endColumn = processedColumns + vectorizedColumns; + + for (int row = 0; row < height; row++) + { + Vector256 leftVector = usesLeft ? Vector256.Create(Unsafe.Add(ref leftBase, row)) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = processedColumns; column < endColumn; column += Vector256.Count) + { + Vector256 top = usesTop ? Vector256.LoadUnsafe(ref topBase, (nuint)column) : default; + ref int columnWeights = ref usesColumnWeight ? ref Unsafe.Add(ref columnWeightBase, column) : ref columnWeightBase; + Vector256 prediction = TOperator.Predict(top, leftVector, topLeftVector, topRightVector, bottomLeftVector, ref columnWeights, rowWeight); + prediction.StoreUnsafe(ref destinationRow, (nuint)column); + } + } + + processedColumns = endColumn; + } + } + + if (Vector128.IsHardwareAccelerated) + { + int remainingColumns = width - processedColumns; + int vectorizedColumns = remainingColumns - (remainingColumns % Vector128.Count); + if (vectorizedColumns > 0) + { + Vector128 topLeftVector = usesTopLeft ? Vector128.Create(topLeft) : default; + Vector128 topRightVector = usesTopRight ? Vector128.Create(topRight) : default; + Vector128 bottomLeftVector = usesBottomLeft ? Vector128.Create(bottomLeft) : default; + int endColumn = processedColumns + vectorizedColumns; + + for (int row = 0; row < height; row++) + { + Vector128 leftVector = usesLeft ? Vector128.Create(Unsafe.Add(ref leftBase, row)) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = processedColumns; column < endColumn; column += Vector128.Count) + { + Vector128 top = usesTop ? Vector128.LoadUnsafe(ref topBase, (nuint)column) : default; + ref int columnWeights = ref usesColumnWeight ? ref Unsafe.Add(ref columnWeightBase, column) : ref columnWeightBase; + Vector128 prediction = TOperator.Predict(top, leftVector, topLeftVector, topRightVector, bottomLeftVector, ref columnWeights, rowWeight); + prediction.StoreUnsafe(ref destinationRow, (nuint)column); + } + } + + processedColumns = endColumn; + } + } + + // AV1 dimensions are multiples of four. The tail is normally zero on SIMD hardware, but retaining the + // scalar continuation keeps the traversal correct when intrinsics are disabled by FeatureTestRunner. + for (int row = 0; row < height; row++) + { + byte leftSample = usesLeft ? Unsafe.Add(ref leftBase, row) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = processedColumns; column < width; column++) + { + byte top = usesTop ? Unsafe.Add(ref topBase, column) : default; + int columnWeight = usesColumnWeight ? Unsafe.Add(ref columnWeightBase, column) : 0; + Unsafe.Add(ref destinationRow, column) = TOperator.Predict(top, leftSample, topLeft, topRight, bottomLeft, columnWeight, rowWeight); + } + } + } + + /// + public override void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height) + { + ref short destinationBase = ref MemoryMarshal.GetReference(destination); + ref short topBase = ref MemoryMarshal.GetReference(above); + ref short leftBase = ref MemoryMarshal.GetReference(left); + ref int columnWeightBase = ref MemoryMarshal.GetReference(SmoothWeights[width..]); + + Av1IntraPredictionInputs inputs = TOperator.Inputs; + bool usesTop = (inputs & Av1IntraPredictionInputs.Top) != 0; + bool usesLeft = (inputs & Av1IntraPredictionInputs.Left) != 0; + bool usesTopLeft = (inputs & Av1IntraPredictionInputs.TopLeft) != 0; + bool usesTopRight = (inputs & Av1IntraPredictionInputs.TopRight) != 0; + bool usesBottomLeft = (inputs & Av1IntraPredictionInputs.BottomLeft) != 0; + bool usesColumnWeight = (inputs & Av1IntraPredictionInputs.ColumnWeight) != 0; + bool usesRowWeight = (inputs & Av1IntraPredictionInputs.RowWeight) != 0; + + short topLeft = usesTopLeft ? Unsafe.Subtract(ref topBase, 1) : default; + short topRight = usesTopRight ? Unsafe.Add(ref topBase, width - 1) : default; + short bottomLeft = usesBottomLeft ? Unsafe.Add(ref leftBase, height - 1) : default; + int processedColumns = 0; + + if (Vector512.IsHardwareAccelerated) + { + int vectorizedColumns = width - (width % Vector512.Count); + if (vectorizedColumns > 0) + { + Vector512 topLeftVector = usesTopLeft ? Vector512.Create(topLeft) : default; + Vector512 topRightVector = usesTopRight ? Vector512.Create(topRight) : default; + Vector512 bottomLeftVector = usesBottomLeft ? Vector512.Create(bottomLeft) : default; + + for (int row = 0; row < height; row++) + { + Vector512 leftVector = usesLeft ? Vector512.Create(Unsafe.Add(ref leftBase, row)) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref short destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < vectorizedColumns; column += Vector512.Count) + { + Vector512 top = usesTop ? Vector512.LoadUnsafe(ref topBase, (nuint)column) : default; + ref int columnWeights = ref usesColumnWeight ? ref Unsafe.Add(ref columnWeightBase, column) : ref columnWeightBase; + Vector512 prediction = TOperator.Predict(top, leftVector, topLeftVector, topRightVector, bottomLeftVector, ref columnWeights, rowWeight); + prediction.StoreUnsafe(ref destinationRow, (nuint)column); + } + } + + processedColumns = vectorizedColumns; + } + } + + if (Vector256.IsHardwareAccelerated) + { + int remainingColumns = width - processedColumns; + int vectorizedColumns = remainingColumns - (remainingColumns % Vector256.Count); + if (vectorizedColumns > 0) + { + Vector256 topLeftVector = usesTopLeft ? Vector256.Create(topLeft) : default; + Vector256 topRightVector = usesTopRight ? Vector256.Create(topRight) : default; + Vector256 bottomLeftVector = usesBottomLeft ? Vector256.Create(bottomLeft) : default; + int endColumn = processedColumns + vectorizedColumns; + + for (int row = 0; row < height; row++) + { + Vector256 leftVector = usesLeft ? Vector256.Create(Unsafe.Add(ref leftBase, row)) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref short destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = processedColumns; column < endColumn; column += Vector256.Count) + { + Vector256 top = usesTop ? Vector256.LoadUnsafe(ref topBase, (nuint)column) : default; + ref int columnWeights = ref usesColumnWeight ? ref Unsafe.Add(ref columnWeightBase, column) : ref columnWeightBase; + Vector256 prediction = TOperator.Predict(top, leftVector, topLeftVector, topRightVector, bottomLeftVector, ref columnWeights, rowWeight); + prediction.StoreUnsafe(ref destinationRow, (nuint)column); + } + } + + processedColumns = endColumn; + } + } + + if (Vector128.IsHardwareAccelerated) + { + int remainingColumns = width - processedColumns; + int vectorizedColumns = remainingColumns - (remainingColumns % Vector128.Count); + if (vectorizedColumns > 0) + { + Vector128 topLeftVector = usesTopLeft ? Vector128.Create(topLeft) : default; + Vector128 topRightVector = usesTopRight ? Vector128.Create(topRight) : default; + Vector128 bottomLeftVector = usesBottomLeft ? Vector128.Create(bottomLeft) : default; + int endColumn = processedColumns + vectorizedColumns; + + for (int row = 0; row < height; row++) + { + Vector128 leftVector = usesLeft ? Vector128.Create(Unsafe.Add(ref leftBase, row)) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref short destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = processedColumns; column < endColumn; column += Vector128.Count) + { + Vector128 top = usesTop ? Vector128.LoadUnsafe(ref topBase, (nuint)column) : default; + ref int columnWeights = ref usesColumnWeight ? ref Unsafe.Add(ref columnWeightBase, column) : ref columnWeightBase; + Vector128 prediction = TOperator.Predict(top, leftVector, topLeftVector, topRightVector, bottomLeftVector, ref columnWeights, rowWeight); + prediction.StoreUnsafe(ref destinationRow, (nuint)column); + } + } + + processedColumns = endColumn; + } + } + + for (int row = 0; row < height; row++) + { + short leftSample = usesLeft ? Unsafe.Add(ref leftBase, row) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref short destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = processedColumns; column < width; column++) + { + short top = usesTop ? Unsafe.Add(ref topBase, column) : default; + int columnWeight = usesColumnWeight ? Unsafe.Add(ref columnWeightBase, column) : 0; + Unsafe.Add(ref destinationRow, column) = TOperator.Predict(top, leftSample, topLeft, topRight, bottomLeft, columnWeight, rowWeight); + } + } + } + + /// + public override void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height) + { + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref byte topBase = ref MemoryMarshal.GetReference(above); + ref byte leftBase = ref MemoryMarshal.GetReference(left); + ref int columnWeightBase = ref MemoryMarshal.GetReference(SmoothWeights[width..]); + + Av1IntraPredictionInputs inputs = TOperator.Inputs; + bool usesTop = (inputs & Av1IntraPredictionInputs.Top) != 0; + bool usesLeft = (inputs & Av1IntraPredictionInputs.Left) != 0; + bool usesTopLeft = (inputs & Av1IntraPredictionInputs.TopLeft) != 0; + bool usesTopRight = (inputs & Av1IntraPredictionInputs.TopRight) != 0; + bool usesBottomLeft = (inputs & Av1IntraPredictionInputs.BottomLeft) != 0; + bool usesColumnWeight = (inputs & Av1IntraPredictionInputs.ColumnWeight) != 0; + bool usesRowWeight = (inputs & Av1IntraPredictionInputs.RowWeight) != 0; + byte topLeft = usesTopLeft ? Unsafe.Subtract(ref topBase, 1) : default; + byte topRight = usesTopRight ? Unsafe.Add(ref topBase, width - 1) : default; + byte bottomLeft = usesBottomLeft ? Unsafe.Add(ref leftBase, height - 1) : default; + + for (int row = 0; row < height; row++) + { + byte leftSample = usesLeft ? Unsafe.Add(ref leftBase, row) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + byte top = usesTop ? Unsafe.Add(ref topBase, column) : default; + int columnWeight = usesColumnWeight ? Unsafe.Add(ref columnWeightBase, column) : 0; + Unsafe.Add(ref destinationRow, column) = TOperator.Predict(top, leftSample, topLeft, topRight, bottomLeft, columnWeight, rowWeight); + } + } + } + + /// + public override void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height) + { + ref short destinationBase = ref MemoryMarshal.GetReference(destination); + ref short topBase = ref MemoryMarshal.GetReference(above); + ref short leftBase = ref MemoryMarshal.GetReference(left); + ref int columnWeightBase = ref MemoryMarshal.GetReference(SmoothWeights[width..]); + + Av1IntraPredictionInputs inputs = TOperator.Inputs; + bool usesTop = (inputs & Av1IntraPredictionInputs.Top) != 0; + bool usesLeft = (inputs & Av1IntraPredictionInputs.Left) != 0; + bool usesTopLeft = (inputs & Av1IntraPredictionInputs.TopLeft) != 0; + bool usesTopRight = (inputs & Av1IntraPredictionInputs.TopRight) != 0; + bool usesBottomLeft = (inputs & Av1IntraPredictionInputs.BottomLeft) != 0; + bool usesColumnWeight = (inputs & Av1IntraPredictionInputs.ColumnWeight) != 0; + bool usesRowWeight = (inputs & Av1IntraPredictionInputs.RowWeight) != 0; + short topLeft = usesTopLeft ? Unsafe.Subtract(ref topBase, 1) : default; + short topRight = usesTopRight ? Unsafe.Add(ref topBase, width - 1) : default; + short bottomLeft = usesBottomLeft ? Unsafe.Add(ref leftBase, height - 1) : default; + + for (int row = 0; row < height; row++) + { + short leftSample = usesLeft ? Unsafe.Add(ref leftBase, row) : default; + int rowWeight = usesRowWeight ? SmoothWeights[height + row] : 0; + ref short destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + short top = usesTop ? Unsafe.Add(ref topBase, column) : default; + int columnWeight = usesColumnWeight ? Unsafe.Add(ref columnWeightBase, column) : 0; + Unsafe.Add(ref destinationRow, column) = TOperator.Predict(top, leftSample, topLeft, topRight, bottomLeft, columnWeight, rowWeight); + } + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs new file mode 100644 index 0000000000..deb1854851 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs @@ -0,0 +1,180 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Identifies the neighboring inputs consumed by an AV1 intra-prediction operator. + /// + [Flags] + internal enum Av1IntraPredictionInputs + { + /// + /// The operator does not consume neighboring samples. + /// + None = 0, + + /// + /// The operator consumes samples from the top reference. + /// + Top = 1, + + /// + /// The operator consumes samples from the left reference. + /// + Left = 2, + + /// + /// The operator consumes the shared top-left reference. + /// + TopLeft = 4, + + /// + /// The operator consumes the final top reference. + /// + TopRight = 8, + + /// + /// The operator consumes the final left reference. + /// + BottomLeft = 16, + + /// + /// The operator consumes the horizontal smooth weights. + /// + ColumnWeight = 32, + + /// + /// The operator consumes the vertical smooth weights. + /// + RowWeight = 64, + } + + /// + /// Defines the scalar and SIMD arithmetic for one non-directional AV1 intra-prediction mode. + /// + /// + /// Each overload performs the same lane-wise operation. The generic predictor traversal selects the widest + /// available overload, and the JIT specializes each static interface call for the closed operator type. + /// + internal interface IAv1IntraPredictionOperator + { + /// + /// Gets the prediction mode implemented by the operator. + /// + public static abstract Av1PredictionMode Mode { get; } + + /// + /// Gets the neighboring inputs consumed by the operator. + /// + public static abstract Av1IntraPredictionInputs Inputs { get; } + + /// + /// Predicts one 8-bit sample when hardware vectorization is unavailable. + /// + /// The top reference sample. + /// The left reference sample. + /// The shared top-left reference sample. + /// The final top reference sample. + /// The final left reference sample. + /// The horizontal Q8 smooth weight. + /// The vertical Q8 smooth weight. + /// The predicted sample. + public static abstract byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight); + + /// + /// Predicts sixteen 8-bit samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts thirty-two 8-bit samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts sixty-four 8-bit samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts one high-bit-depth sample when hardware vectorization is unavailable. + /// + /// The top reference sample. + /// The left reference sample. + /// The shared top-left reference sample. + /// The final top reference sample. + /// The final left reference sample. + /// The horizontal Q8 smooth weight. + /// The vertical Q8 smooth weight. + /// The predicted sample. + public static abstract short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight); + + /// + /// Predicts eight high-bit-depth samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts sixteen high-bit-depth samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts thirty-two high-bit-depth samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Packing.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Packing.cs new file mode 100644 index 0000000000..a0faad29d0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Packing.cs @@ -0,0 +1,166 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Widens AV1 sample vectors for fixed-point prediction arithmetic and narrows completed results. +/// +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Widens sixteen 8-bit samples into four 32-bit vectors. + /// + /// The packed samples. + /// The first four widened samples. + /// The second four widened samples. + /// The third four widened samples. + /// The fourth four widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Widen(Vector128 source, out Vector128 result0, out Vector128 result1, out Vector128 result2, out Vector128 result3) + { + (Vector128 low, Vector128 high) = Vector128.Widen(source); + (Vector128 low0, Vector128 low1) = Vector128.Widen(low); + (Vector128 high0, Vector128 high1) = Vector128.Widen(high); + result0 = low0.AsInt32(); + result1 = low1.AsInt32(); + result2 = high0.AsInt32(); + result3 = high1.AsInt32(); + } + + /// + /// Widens thirty-two 8-bit samples into four 32-bit vectors. + /// + /// The packed samples. + /// The first eight widened samples. + /// The second eight widened samples. + /// The third eight widened samples. + /// The fourth eight widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Widen(Vector256 source, out Vector256 result0, out Vector256 result1, out Vector256 result2, out Vector256 result3) + { + (Vector256 low, Vector256 high) = Vector256.Widen(source); + (Vector256 low0, Vector256 low1) = Vector256.Widen(low); + (Vector256 high0, Vector256 high1) = Vector256.Widen(high); + result0 = low0.AsInt32(); + result1 = low1.AsInt32(); + result2 = high0.AsInt32(); + result3 = high1.AsInt32(); + } + + /// + /// Widens sixty-four 8-bit samples into four 32-bit vectors. + /// + /// The packed samples. + /// The first sixteen widened samples. + /// The second sixteen widened samples. + /// The third sixteen widened samples. + /// The fourth sixteen widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Widen(Vector512 source, out Vector512 result0, out Vector512 result1, out Vector512 result2, out Vector512 result3) + { + (Vector512 low, Vector512 high) = Vector512.Widen(source); + (Vector512 low0, Vector512 low1) = Vector512.Widen(low); + (Vector512 high0, Vector512 high1) = Vector512.Widen(high); + result0 = low0.AsInt32(); + result1 = low1.AsInt32(); + result2 = high0.AsInt32(); + result3 = high1.AsInt32(); + } + + /// + /// Widens eight high-bit-depth samples into two 32-bit vectors. + /// + /// The packed samples. + /// The first four widened samples. + /// The second four widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Widen(Vector128 source, out Vector128 result0, out Vector128 result1) + => (result0, result1) = Vector128.Widen(source); + + /// + /// Widens sixteen high-bit-depth samples into two 32-bit vectors. + /// + /// The packed samples. + /// The first eight widened samples. + /// The second eight widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Widen(Vector256 source, out Vector256 result0, out Vector256 result1) + => (result0, result1) = Vector256.Widen(source); + + /// + /// Widens thirty-two high-bit-depth samples into two 32-bit vectors. + /// + /// The packed samples. + /// The first sixteen widened samples. + /// The second sixteen widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Widen(Vector512 source, out Vector512 result0, out Vector512 result1) + => (result0, result1) = Vector512.Widen(source); + + /// + /// Narrows four 32-bit vectors into sixteen 8-bit samples. + /// + /// The first four samples. + /// The second four samples. + /// The third four samples. + /// The fourth four samples. + /// The packed samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Narrow(Vector128 source0, Vector128 source1, Vector128 source2, Vector128 source3) + => Vector128.Narrow(Vector128.Narrow(source0.AsUInt32(), source1.AsUInt32()), Vector128.Narrow(source2.AsUInt32(), source3.AsUInt32())); + + /// + /// Narrows four 32-bit vectors into thirty-two 8-bit samples. + /// + /// The first eight samples. + /// The second eight samples. + /// The third eight samples. + /// The fourth eight samples. + /// The packed samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Narrow(Vector256 source0, Vector256 source1, Vector256 source2, Vector256 source3) + => Vector256.Narrow(Vector256.Narrow(source0.AsUInt32(), source1.AsUInt32()), Vector256.Narrow(source2.AsUInt32(), source3.AsUInt32())); + + /// + /// Narrows four 32-bit vectors into sixty-four 8-bit samples. + /// + /// The first sixteen samples. + /// The second sixteen samples. + /// The third sixteen samples. + /// The fourth sixteen samples. + /// The packed samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Narrow(Vector512 source0, Vector512 source1, Vector512 source2, Vector512 source3) + => Vector512.Narrow(Vector512.Narrow(source0.AsUInt32(), source1.AsUInt32()), Vector512.Narrow(source2.AsUInt32(), source3.AsUInt32())); + + /// + /// Narrows two 32-bit vectors into eight high-bit-depth samples. + /// + /// The first four samples. + /// The second four samples. + /// The packed samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Narrow(Vector128 source0, Vector128 source1) => Vector128.Narrow(source0, source1); + + /// + /// Narrows two 32-bit vectors into sixteen high-bit-depth samples. + /// + /// The first eight samples. + /// The second eight samples. + /// The packed samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Narrow(Vector256 source0, Vector256 source1) => Vector256.Narrow(source0, source1); + + /// + /// Narrows two 32-bit vectors into thirty-two high-bit-depth samples. + /// + /// The first sixteen samples. + /// The second sixteen samples. + /// The packed samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Narrow(Vector512 source0, Vector512 source1) => Vector512.Narrow(source0, source1); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.PaethOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.PaethOperator.cs new file mode 100644 index 0000000000..05277e1c38 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.PaethOperator.cs @@ -0,0 +1,209 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Implements AV1 Paeth intra prediction for scalar and SIMD lanes. + /// + internal readonly struct PaethOperator : IAv1IntraPredictionOperator + { + /// + public static Av1PredictionMode Mode => Av1PredictionMode.Paeth; + + /// + public static Av1IntraPredictionInputs Inputs + => Av1IntraPredictionInputs.Top | Av1IntraPredictionInputs.Left | Av1IntraPredictionInputs.TopLeft; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight) + { + int basis = top + left - topLeft; + int distanceLeft = Math.Abs(basis - left); + int distanceTop = Math.Abs(basis - top); + int distanceTopLeft = Math.Abs(basis - topLeft); + + // AV1 resolves equal distances in left, top, top-left order. + return distanceLeft <= distanceTop && distanceLeft <= distanceTopLeft ? left : distanceTop <= distanceTopLeft ? top : topLeft; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + => PredictPaeth(top, left, topLeft); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + => PredictPaeth(top, left, topLeft); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + => PredictPaeth(top, left, topLeft); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight) + { + int basis = top + left - topLeft; + int distanceLeft = Math.Abs(basis - left); + int distanceTop = Math.Abs(basis - top); + int distanceTopLeft = Math.Abs(basis - topLeft); + + // High-bit-depth prediction has the same tie order as the 8-bit process. + return distanceLeft <= distanceTop && distanceLeft <= distanceTopLeft ? left : distanceTop <= distanceTopLeft ? top : topLeft; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + => PredictPaeth(top, left, topLeft); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + => PredictPaeth(top, left, topLeft); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + => PredictPaeth(top, left, topLeft); + + /// + /// Selects the nearest Paeth reference for sixteen 8-bit lanes. + /// + /// The top candidates. + /// The left candidates. + /// The top-left candidates. + /// The selected candidates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PredictPaeth(Vector128 top, Vector128 left, Vector128 topLeft) + { + // The Paeth distances to left and top simplify to |top - topLeft| and |left - topLeft|. The established + // PNG predictor uses the same byte-lane identity, avoiding four widening stages for every sixteen samples. + Vector128 topMinusTopLeft = Vector128.SubtractSaturate(top, topLeft); + Vector128 leftMinusTopLeft = Vector128.SubtractSaturate(left, topLeft); + Vector128 distanceLeft = Vector128.SubtractSaturate(topLeft, top) | topMinusTopLeft; + Vector128 distanceTop = Vector128.SubtractSaturate(topLeft, left) | leftMinusTopLeft; + Vector128 sameDirection = Vector128.Equals(Vector128.Equals(topMinusTopLeft, Vector128.Zero), Vector128.Equals(leftMinusTopLeft, Vector128.Zero)); + Vector128 distanceTopLeft = sameDirection | Vector128.SubtractSaturate(distanceTop, distanceLeft) | Vector128.SubtractSaturate(distanceLeft, distanceTop); + Vector128 minimumTopTopLeft = Vector128.Min(distanceTopLeft, distanceTop); + Vector128 topOrTopLeft = Vector128.ConditionalSelect(Vector128.Equals(minimumTopTopLeft, distanceTop), top, topLeft); + return Vector128.ConditionalSelect(Vector128.Equals(Vector128.Min(minimumTopTopLeft, distanceLeft), distanceLeft), left, topOrTopLeft); + } + + /// + /// Selects the nearest Paeth reference for thirty-two 8-bit lanes. + /// + /// The top candidates. + /// The left candidates. + /// The top-left candidates. + /// The selected candidates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PredictPaeth(Vector256 top, Vector256 left, Vector256 topLeft) + { + Vector256 topMinusTopLeft = Vector256.SubtractSaturate(top, topLeft); + Vector256 leftMinusTopLeft = Vector256.SubtractSaturate(left, topLeft); + Vector256 distanceLeft = Vector256.SubtractSaturate(topLeft, top) | topMinusTopLeft; + Vector256 distanceTop = Vector256.SubtractSaturate(topLeft, left) | leftMinusTopLeft; + Vector256 sameDirection = Vector256.Equals(Vector256.Equals(topMinusTopLeft, Vector256.Zero), Vector256.Equals(leftMinusTopLeft, Vector256.Zero)); + Vector256 distanceTopLeft = sameDirection | Vector256.SubtractSaturate(distanceTop, distanceLeft) | Vector256.SubtractSaturate(distanceLeft, distanceTop); + Vector256 minimumTopTopLeft = Vector256.Min(distanceTopLeft, distanceTop); + Vector256 topOrTopLeft = Vector256.ConditionalSelect(Vector256.Equals(minimumTopTopLeft, distanceTop), top, topLeft); + return Vector256.ConditionalSelect(Vector256.Equals(Vector256.Min(minimumTopTopLeft, distanceLeft), distanceLeft), left, topOrTopLeft); + } + + /// + /// Selects the nearest Paeth reference for sixty-four 8-bit lanes. + /// + /// The top candidates. + /// The left candidates. + /// The top-left candidates. + /// The selected candidates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 PredictPaeth(Vector512 top, Vector512 left, Vector512 topLeft) + { + Vector512 topMinusTopLeft = Vector512.SubtractSaturate(top, topLeft); + Vector512 leftMinusTopLeft = Vector512.SubtractSaturate(left, topLeft); + Vector512 distanceLeft = Vector512.SubtractSaturate(topLeft, top) | topMinusTopLeft; + Vector512 distanceTop = Vector512.SubtractSaturate(topLeft, left) | leftMinusTopLeft; + Vector512 sameDirection = Vector512.Equals(Vector512.Equals(topMinusTopLeft, Vector512.Zero), Vector512.Equals(leftMinusTopLeft, Vector512.Zero)); + Vector512 distanceTopLeft = sameDirection | Vector512.SubtractSaturate(distanceTop, distanceLeft) | Vector512.SubtractSaturate(distanceLeft, distanceTop); + Vector512 minimumTopTopLeft = Vector512.Min(distanceTopLeft, distanceTop); + Vector512 topOrTopLeft = Vector512.ConditionalSelect(Vector512.Equals(minimumTopTopLeft, distanceTop), top, topLeft); + return Vector512.ConditionalSelect(Vector512.Equals(Vector512.Min(minimumTopTopLeft, distanceLeft), distanceLeft), left, topOrTopLeft); + } + + /// + /// Selects the nearest Paeth reference for eight high-bit-depth lanes. + /// + /// The top candidates. + /// The left candidates. + /// The top-left candidates. + /// The selected candidates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PredictPaeth(Vector128 top, Vector128 left, Vector128 topLeft) + { + // AV1 samples are at most twelve bits. Both signed differences and their sum are therefore bounded by + // 8190, allowing the exact Paeth distances to stay in signed 16-bit lanes without widening. + Vector128 topDelta = top - topLeft; + Vector128 leftDelta = left - topLeft; + Vector128 distanceLeft = Vector128.Abs(topDelta); + Vector128 distanceTop = Vector128.Abs(leftDelta); + Vector128 distanceTopLeft = Vector128.Abs(topDelta + leftDelta); + + Vector128 minimumTopTopLeft = Vector128.Min(distanceTopLeft, distanceTop); + Vector128 topOrTopLeft = Vector128.ConditionalSelect(Vector128.Equals(minimumTopTopLeft, distanceTop), top, topLeft); + return Vector128.ConditionalSelect(Vector128.Equals(Vector128.Min(minimumTopTopLeft, distanceLeft), distanceLeft), left, topOrTopLeft); + } + + /// + /// Selects the nearest Paeth reference for sixteen high-bit-depth lanes. + /// + /// The top candidates. + /// The left candidates. + /// The top-left candidates. + /// The selected candidates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PredictPaeth(Vector256 top, Vector256 left, Vector256 topLeft) + { + Vector256 topDelta = top - topLeft; + Vector256 leftDelta = left - topLeft; + Vector256 distanceLeft = Vector256.Abs(topDelta); + Vector256 distanceTop = Vector256.Abs(leftDelta); + Vector256 distanceTopLeft = Vector256.Abs(topDelta + leftDelta); + + Vector256 minimumTopTopLeft = Vector256.Min(distanceTopLeft, distanceTop); + Vector256 topOrTopLeft = Vector256.ConditionalSelect(Vector256.Equals(minimumTopTopLeft, distanceTop), top, topLeft); + return Vector256.ConditionalSelect(Vector256.Equals(Vector256.Min(minimumTopTopLeft, distanceLeft), distanceLeft), left, topOrTopLeft); + } + + /// + /// Selects the nearest Paeth reference for thirty-two high-bit-depth lanes. + /// + /// The top candidates. + /// The left candidates. + /// The top-left candidates. + /// The selected candidates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 PredictPaeth(Vector512 top, Vector512 left, Vector512 topLeft) + { + Vector512 topDelta = top - topLeft; + Vector512 leftDelta = left - topLeft; + Vector512 distanceLeft = Vector512.Abs(topDelta); + Vector512 distanceTop = Vector512.Abs(leftDelta); + Vector512 distanceTopLeft = Vector512.Abs(topDelta + leftDelta); + + Vector512 minimumTopTopLeft = Vector512.Min(distanceTopLeft, distanceTop); + Vector512 topOrTopLeft = Vector512.ConditionalSelect(Vector512.Equals(minimumTopTopLeft, distanceTop), top, topLeft); + return Vector512.ConditionalSelect(Vector512.Equals(Vector512.Min(minimumTopTopLeft, distanceLeft), distanceLeft), left, topOrTopLeft); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothHorizontalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothHorizontalOperator.cs new file mode 100644 index 0000000000..9256b69471 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothHorizontalOperator.cs @@ -0,0 +1,161 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Implements horizontal AV1 smooth intra prediction for scalar and SIMD lanes. + /// + internal readonly struct SmoothHorizontalOperator : IAv1IntraPredictionOperator + { + /// + /// The Q8 scale used by the smooth surface. + /// + private const int WeightScale = 256; + + /// + public static Av1PredictionMode Mode => Av1PredictionMode.SmoothHorizontal; + + /// + public static Av1IntraPredictionInputs Inputs + => Av1IntraPredictionInputs.Left | Av1IntraPredictionInputs.TopRight | Av1IntraPredictionInputs.ColumnWeight; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight) + => (byte)(((left * columnWeight) + (topRight * (WeightScale - columnWeight)) + (WeightScale >> 1)) >> 8); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + { + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector128 endpointBias = Vector128.Create((topRightSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(Vector128.LoadUnsafe(ref columnWeights), horizontalDelta, endpointBias), + Calculate(Vector128.LoadUnsafe(ref columnWeights, (nuint)Vector128.Count), horizontalDelta, endpointBias), + Calculate(Vector128.LoadUnsafe(ref columnWeights, (nuint)(2 * Vector128.Count)), horizontalDelta, endpointBias), + Calculate(Vector128.LoadUnsafe(ref columnWeights, (nuint)(3 * Vector128.Count)), horizontalDelta, endpointBias)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + { + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector256 endpointBias = Vector256.Create((topRightSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(Vector256.LoadUnsafe(ref columnWeights), horizontalDelta, endpointBias), + Calculate(Vector256.LoadUnsafe(ref columnWeights, (nuint)Vector256.Count), horizontalDelta, endpointBias), + Calculate(Vector256.LoadUnsafe(ref columnWeights, (nuint)(2 * Vector256.Count)), horizontalDelta, endpointBias), + Calculate(Vector256.LoadUnsafe(ref columnWeights, (nuint)(3 * Vector256.Count)), horizontalDelta, endpointBias)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + { + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector512 endpointBias = Vector512.Create((topRightSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(Vector512.LoadUnsafe(ref columnWeights), horizontalDelta, endpointBias), + Calculate(Vector512.LoadUnsafe(ref columnWeights, (nuint)Vector512.Count), horizontalDelta, endpointBias), + Calculate(Vector512.LoadUnsafe(ref columnWeights, (nuint)(2 * Vector512.Count)), horizontalDelta, endpointBias), + Calculate(Vector512.LoadUnsafe(ref columnWeights, (nuint)(3 * Vector512.Count)), horizontalDelta, endpointBias)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight) + => (short)(((left * columnWeight) + (topRight * (WeightScale - columnWeight)) + (WeightScale >> 1)) >> 8); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + { + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector128 endpointBias = Vector128.Create((topRightSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(Vector128.LoadUnsafe(ref columnWeights), horizontalDelta, endpointBias), + Calculate(Vector128.LoadUnsafe(ref columnWeights, (nuint)Vector128.Count), horizontalDelta, endpointBias)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + { + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector256 endpointBias = Vector256.Create((topRightSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(Vector256.LoadUnsafe(ref columnWeights), horizontalDelta, endpointBias), + Calculate(Vector256.LoadUnsafe(ref columnWeights, (nuint)Vector256.Count), horizontalDelta, endpointBias)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + { + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector512 endpointBias = Vector512.Create((topRightSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(Vector512.LoadUnsafe(ref columnWeights), horizontalDelta, endpointBias), + Calculate(Vector512.LoadUnsafe(ref columnWeights, (nuint)Vector512.Count), horizontalDelta, endpointBias)); + } + + /// + /// Calculates four horizontal smooth predictions. + /// + /// The horizontal Q8 weights. + /// The difference between the left sample and top-right endpoint. + /// The top-right endpoint and rounding bias in every lane. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Calculate(Vector128 columnWeights, int horizontalDelta, Vector128 endpointBias) + => ((columnWeights * horizontalDelta) + endpointBias) >> 8; + + /// + /// Calculates eight horizontal smooth predictions. + /// + /// The horizontal Q8 weights. + /// The difference between the left sample and top-right endpoint. + /// The top-right endpoint and rounding bias in every lane. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Calculate(Vector256 columnWeights, int horizontalDelta, Vector256 endpointBias) + => ((columnWeights * horizontalDelta) + endpointBias) >> 8; + + /// + /// Calculates sixteen horizontal smooth predictions. + /// + /// The horizontal Q8 weights. + /// The difference between the left sample and top-right endpoint. + /// The top-right endpoint and rounding bias in every lane. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Calculate(Vector512 columnWeights, int horizontalDelta, Vector512 endpointBias) + => ((columnWeights * horizontalDelta) + endpointBias) >> 8; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothOperator.cs new file mode 100644 index 0000000000..d56183ff17 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothOperator.cs @@ -0,0 +1,241 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Implements two-dimensional AV1 smooth intra prediction for scalar and SIMD lanes. + /// + internal readonly struct SmoothOperator : IAv1IntraPredictionOperator + { + /// + /// The Q8 scale used by each smooth surface. + /// + private const int WeightScale = 256; + + /// + public static Av1PredictionMode Mode => Av1PredictionMode.Smooth; + + /// + public static Av1IntraPredictionInputs Inputs + => Av1IntraPredictionInputs.Top + | Av1IntraPredictionInputs.Left + | Av1IntraPredictionInputs.TopRight + | Av1IntraPredictionInputs.BottomLeft + | Av1IntraPredictionInputs.ColumnWeight + | Av1IntraPredictionInputs.RowWeight; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight) + { + int prediction = (top * rowWeight) + (bottomLeft * (WeightScale - rowWeight)); + prediction += (left * columnWeight) + (topRight * (WeightScale - columnWeight)); + return (byte)((prediction + WeightScale) >> 9); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector128 top0, out Vector128 top1, out Vector128 top2, out Vector128 top3); + Vector128 weight0 = Vector128.LoadUnsafe(ref columnWeights); + Vector128 weight1 = Vector128.LoadUnsafe(ref columnWeights, (nuint)Vector128.Count); + Vector128 weight2 = Vector128.LoadUnsafe(ref columnWeights, (nuint)(2 * Vector128.Count)); + Vector128 weight3 = Vector128.LoadUnsafe(ref columnWeights, (nuint)(3 * Vector128.Count)); + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int bottomLeftSample = bottomLeft.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector128 bottomLeftVector = Vector128.Create(bottomLeftSample); + Vector128 endpointBias = Vector128.Create(((bottomLeftSample + topRightSample) * WeightScale) + WeightScale); + + return Narrow( + Calculate(top0, weight0, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top1, weight1, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top2, weight2, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top3, weight3, bottomLeftVector, horizontalDelta, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector256 top0, out Vector256 top1, out Vector256 top2, out Vector256 top3); + Vector256 weight0 = Vector256.LoadUnsafe(ref columnWeights); + Vector256 weight1 = Vector256.LoadUnsafe(ref columnWeights, (nuint)Vector256.Count); + Vector256 weight2 = Vector256.LoadUnsafe(ref columnWeights, (nuint)(2 * Vector256.Count)); + Vector256 weight3 = Vector256.LoadUnsafe(ref columnWeights, (nuint)(3 * Vector256.Count)); + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int bottomLeftSample = bottomLeft.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector256 bottomLeftVector = Vector256.Create(bottomLeftSample); + Vector256 endpointBias = Vector256.Create(((bottomLeftSample + topRightSample) * WeightScale) + WeightScale); + + return Narrow( + Calculate(top0, weight0, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top1, weight1, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top2, weight2, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top3, weight3, bottomLeftVector, horizontalDelta, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector512 top0, out Vector512 top1, out Vector512 top2, out Vector512 top3); + Vector512 weight0 = Vector512.LoadUnsafe(ref columnWeights); + Vector512 weight1 = Vector512.LoadUnsafe(ref columnWeights, (nuint)Vector512.Count); + Vector512 weight2 = Vector512.LoadUnsafe(ref columnWeights, (nuint)(2 * Vector512.Count)); + Vector512 weight3 = Vector512.LoadUnsafe(ref columnWeights, (nuint)(3 * Vector512.Count)); + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int bottomLeftSample = bottomLeft.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector512 bottomLeftVector = Vector512.Create(bottomLeftSample); + Vector512 endpointBias = Vector512.Create(((bottomLeftSample + topRightSample) * WeightScale) + WeightScale); + + return Narrow( + Calculate(top0, weight0, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top1, weight1, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top2, weight2, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top3, weight3, bottomLeftVector, horizontalDelta, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight) + { + int prediction = (top * rowWeight) + (bottomLeft * (WeightScale - rowWeight)); + prediction += (left * columnWeight) + (topRight * (WeightScale - columnWeight)); + return (short)((prediction + WeightScale) >> 9); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector128 top0, out Vector128 top1); + Vector128 weight0 = Vector128.LoadUnsafe(ref columnWeights); + Vector128 weight1 = Vector128.LoadUnsafe(ref columnWeights, (nuint)Vector128.Count); + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int bottomLeftSample = bottomLeft.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector128 bottomLeftVector = Vector128.Create(bottomLeftSample); + Vector128 endpointBias = Vector128.Create(((bottomLeftSample + topRightSample) * WeightScale) + WeightScale); + + return Narrow( + Calculate(top0, weight0, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top1, weight1, bottomLeftVector, horizontalDelta, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector256 top0, out Vector256 top1); + Vector256 weight0 = Vector256.LoadUnsafe(ref columnWeights); + Vector256 weight1 = Vector256.LoadUnsafe(ref columnWeights, (nuint)Vector256.Count); + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int bottomLeftSample = bottomLeft.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector256 bottomLeftVector = Vector256.Create(bottomLeftSample); + Vector256 endpointBias = Vector256.Create(((bottomLeftSample + topRightSample) * WeightScale) + WeightScale); + + return Narrow( + Calculate(top0, weight0, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top1, weight1, bottomLeftVector, horizontalDelta, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector512 top0, out Vector512 top1); + Vector512 weight0 = Vector512.LoadUnsafe(ref columnWeights); + Vector512 weight1 = Vector512.LoadUnsafe(ref columnWeights, (nuint)Vector512.Count); + int leftSample = left.GetElement(0); + int topRightSample = topRight.GetElement(0); + int bottomLeftSample = bottomLeft.GetElement(0); + int horizontalDelta = leftSample - topRightSample; + Vector512 bottomLeftVector = Vector512.Create(bottomLeftSample); + Vector512 endpointBias = Vector512.Create(((bottomLeftSample + topRightSample) * WeightScale) + WeightScale); + + return Narrow( + Calculate(top0, weight0, bottomLeftVector, horizontalDelta, endpointBias, rowWeight), + Calculate(top1, weight1, bottomLeftVector, horizontalDelta, endpointBias, rowWeight)); + } + + /// + /// Calculates four two-dimensional smooth predictions. + /// + /// The widened top samples. + /// The horizontal Q8 weights. + /// The bottom-left endpoint in every lane. + /// The difference between the left sample and top-right endpoint. + /// The combined endpoint and rounding bias in every lane. + /// The vertical Q8 weight. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Calculate( + Vector128 top, + Vector128 columnWeights, + Vector128 bottomLeft, + int horizontalDelta, + Vector128 endpointBias, + int rowWeight) + { + // Expanding the complementary weights gives the exact normative sum while reducing it to two vector + // multiplications: (top - bottom) * rowWeight + (left - right) * columnWeight + the endpoint bias. + return (((top - bottomLeft) * rowWeight) + (columnWeights * horizontalDelta) + endpointBias) >> 9; + } + + /// + /// Calculates eight two-dimensional smooth predictions. + /// + /// The widened top samples. + /// The horizontal Q8 weights. + /// The bottom-left endpoint in every lane. + /// The difference between the left sample and top-right endpoint. + /// The combined endpoint and rounding bias in every lane. + /// The vertical Q8 weight. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Calculate( + Vector256 top, + Vector256 columnWeights, + Vector256 bottomLeft, + int horizontalDelta, + Vector256 endpointBias, + int rowWeight) + => (((top - bottomLeft) * rowWeight) + (columnWeights * horizontalDelta) + endpointBias) >> 9; + + /// + /// Calculates sixteen two-dimensional smooth predictions. + /// + /// The widened top samples. + /// The horizontal Q8 weights. + /// The bottom-left endpoint in every lane. + /// The difference between the left sample and top-right endpoint. + /// The combined endpoint and rounding bias in every lane. + /// The vertical Q8 weight. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Calculate( + Vector512 top, + Vector512 columnWeights, + Vector512 bottomLeft, + int horizontalDelta, + Vector512 endpointBias, + int rowWeight) + => (((top - bottomLeft) * rowWeight) + (columnWeights * horizontalDelta) + endpointBias) >> 9; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothVerticalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothVerticalOperator.cs new file mode 100644 index 0000000000..37f6dedb7a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothVerticalOperator.cs @@ -0,0 +1,164 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Implements vertical AV1 smooth intra prediction for scalar and SIMD lanes. + /// + internal readonly struct SmoothVerticalOperator : IAv1IntraPredictionOperator + { + /// + /// The Q8 scale used by the smooth surface. + /// + private const int WeightScale = 256; + + /// + public static Av1PredictionMode Mode => Av1PredictionMode.SmoothVertical; + + /// + public static Av1IntraPredictionInputs Inputs + => Av1IntraPredictionInputs.Top | Av1IntraPredictionInputs.BottomLeft | Av1IntraPredictionInputs.RowWeight; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight) + => (byte)(((top * rowWeight) + (bottomLeft * (WeightScale - rowWeight)) + (WeightScale >> 1)) >> 8); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector128 top0, out Vector128 top1, out Vector128 top2, out Vector128 top3); + int bottomLeftSample = bottomLeft.GetElement(0); + Vector128 bottomLeftVector = Vector128.Create(bottomLeftSample); + Vector128 endpointBias = Vector128.Create((bottomLeftSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(top0, bottomLeftVector, endpointBias, rowWeight), + Calculate(top1, bottomLeftVector, endpointBias, rowWeight), + Calculate(top2, bottomLeftVector, endpointBias, rowWeight), + Calculate(top3, bottomLeftVector, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector256 top0, out Vector256 top1, out Vector256 top2, out Vector256 top3); + int bottomLeftSample = bottomLeft.GetElement(0); + Vector256 bottomLeftVector = Vector256.Create(bottomLeftSample); + Vector256 endpointBias = Vector256.Create((bottomLeftSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(top0, bottomLeftVector, endpointBias, rowWeight), + Calculate(top1, bottomLeftVector, endpointBias, rowWeight), + Calculate(top2, bottomLeftVector, endpointBias, rowWeight), + Calculate(top3, bottomLeftVector, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector512 top0, out Vector512 top1, out Vector512 top2, out Vector512 top3); + int bottomLeftSample = bottomLeft.GetElement(0); + Vector512 bottomLeftVector = Vector512.Create(bottomLeftSample); + Vector512 endpointBias = Vector512.Create((bottomLeftSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(top0, bottomLeftVector, endpointBias, rowWeight), + Calculate(top1, bottomLeftVector, endpointBias, rowWeight), + Calculate(top2, bottomLeftVector, endpointBias, rowWeight), + Calculate(top3, bottomLeftVector, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight) + => (short)(((top * rowWeight) + (bottomLeft * (WeightScale - rowWeight)) + (WeightScale >> 1)) >> 8); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector128 top0, out Vector128 top1); + int bottomLeftSample = bottomLeft.GetElement(0); + Vector128 bottomLeftVector = Vector128.Create(bottomLeftSample); + Vector128 endpointBias = Vector128.Create((bottomLeftSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(top0, bottomLeftVector, endpointBias, rowWeight), + Calculate(top1, bottomLeftVector, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector256 top0, out Vector256 top1); + int bottomLeftSample = bottomLeft.GetElement(0); + Vector256 bottomLeftVector = Vector256.Create(bottomLeftSample); + Vector256 endpointBias = Vector256.Create((bottomLeftSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(top0, bottomLeftVector, endpointBias, rowWeight), + Calculate(top1, bottomLeftVector, endpointBias, rowWeight)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) + { + Widen(top, out Vector512 top0, out Vector512 top1); + int bottomLeftSample = bottomLeft.GetElement(0); + Vector512 bottomLeftVector = Vector512.Create(bottomLeftSample); + Vector512 endpointBias = Vector512.Create((bottomLeftSample * WeightScale) + (WeightScale >> 1)); + + return Narrow( + Calculate(top0, bottomLeftVector, endpointBias, rowWeight), + Calculate(top1, bottomLeftVector, endpointBias, rowWeight)); + } + + /// + /// Calculates four vertical smooth predictions. + /// + /// The widened top samples. + /// The bottom-left endpoint in every lane. + /// The bottom-left endpoint and rounding bias in every lane. + /// The vertical Q8 weight. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Calculate(Vector128 top, Vector128 bottomLeft, Vector128 endpointBias, int rowWeight) + => (((top - bottomLeft) * rowWeight) + endpointBias) >> 8; + + /// + /// Calculates eight vertical smooth predictions. + /// + /// The widened top samples. + /// The bottom-left endpoint in every lane. + /// The bottom-left endpoint and rounding bias in every lane. + /// The vertical Q8 weight. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Calculate(Vector256 top, Vector256 bottomLeft, Vector256 endpointBias, int rowWeight) + => (((top - bottomLeft) * rowWeight) + endpointBias) >> 8; + + /// + /// Calculates sixteen vertical smooth predictions. + /// + /// The widened top samples. + /// The bottom-left endpoint in every lane. + /// The bottom-left endpoint and rounding bias in every lane. + /// The vertical Q8 weight. + /// The rounded predictions. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Calculate(Vector512 top, Vector512 bottomLeft, Vector512 endpointBias, int rowWeight) + => (((top - bottomLeft) * rowWeight) + endpointBias) >> 8; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.VerticalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.VerticalOperator.cs new file mode 100644 index 0000000000..954767caba --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.VerticalOperator.cs @@ -0,0 +1,54 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// Implements AV1 vertical intra prediction for scalar and SIMD lanes. + /// + internal readonly struct VerticalOperator : IAv1IntraPredictionOperator + { + /// + public static Av1PredictionMode Mode => Av1PredictionMode.Vertical; + + /// + public static Av1IntraPredictionInputs Inputs => Av1IntraPredictionInputs.Top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight) => top; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight) => top; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictorBase.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictorBase.cs new file mode 100644 index 0000000000..9bb6bc64c5 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictorBase.cs @@ -0,0 +1,129 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Reconstructs non-directional AV1 intra-prediction blocks from prepared neighboring samples. +/// +/// +/// The implementation covers the non-directional prediction processes in section 7.11.2 of the AV1 specification. +/// +internal abstract partial class Av1IntraPredictorBase +{ + /// + /// The horizontal prediction operator. + /// + private static readonly Av1IntraPredictor HorizontalPredictor = new(); + + /// + /// The vertical prediction operator. + /// + private static readonly Av1IntraPredictor VerticalPredictor = new(); + + /// + /// The Paeth prediction operator. + /// + private static readonly Av1IntraPredictor PaethPredictor = new(); + + /// + /// The two-dimensional smooth prediction operator. + /// + private static readonly Av1IntraPredictor SmoothPredictor = new(); + + /// + /// The horizontal smooth prediction operator. + /// + private static readonly Av1IntraPredictor SmoothHorizontalPredictor = new(); + + /// + /// The vertical smooth prediction operator. + /// + private static readonly Av1IntraPredictor SmoothVerticalPredictor = new(); + + /// + /// Gets the Q8 smooth weights for every supported block dimension. + /// + private static ReadOnlySpan SmoothWeights => + [ + + // The first two entries are unused because the smallest AV1 prediction dimension is four samples. + 0, 0, + 255, 128, + 255, 149, 85, 64, + 255, 197, 146, 105, 73, 50, 37, 32, + 255, 225, 196, 170, 145, 123, 102, 84, 68, 54, 43, 33, 26, 20, 17, 16, + 255, 240, 225, 210, 196, 182, 169, 157, 145, 133, 122, 111, 101, 92, 83, 74, + 66, 59, 52, 45, 39, 34, 29, 25, 21, 17, 14, 12, 10, 9, 8, 8, + 255, 248, 240, 233, 225, 218, 210, 203, 196, 189, 182, 176, 169, 163, 156, + 150, 144, 138, 133, 127, 121, 116, 111, 106, 101, 96, 91, 86, 82, 77, 73, 69, + 65, 61, 57, 54, 50, 47, 44, 41, 38, 35, 32, 29, 27, 25, 22, 20, 18, 16, 15, + 13, 12, 10, 9, 8, 7, 6, 6, 5, 5, 4, 4, 4, + ]; + + /// + /// Gets the prediction mode implemented by this predictor. + /// + public abstract Av1PredictionMode Mode { get; } + + /// + /// Gets the closed predictor for a non-directional AV1 prediction mode. + /// + /// The decoded non-directional prediction mode. + /// The predictor for . + public static Av1IntraPredictorBase GetPredictor(Av1PredictionMode mode) + => mode switch + { + Av1PredictionMode.Horizontal => HorizontalPredictor, + Av1PredictionMode.Vertical => VerticalPredictor, + Av1PredictionMode.Paeth => PaethPredictor, + Av1PredictionMode.Smooth => SmoothPredictor, + Av1PredictionMode.SmoothHorizontal => SmoothHorizontalPredictor, + Av1PredictionMode.SmoothVertical => SmoothVerticalPredictor, + _ => throw new InvalidImageContentException($"Prediction mode {mode} is not a non-directional AV1 mode."), + }; + + /// + /// Predicts an 8-bit block. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + public abstract void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height); + + /// + /// Predicts a high-bit-depth block. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + public abstract void Predict(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height); + + /// + /// Predicts an 8-bit block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + public abstract void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height); + + /// + /// Predicts a high-bit-depth block without hardware intrinsics. + /// + /// The destination block origin. + /// The destination row stride in samples. + /// The prepared top reference, preceded in memory by the top-left sample. + /// The prepared left reference. + /// The block width in samples. + /// The block height in samples. + public abstract void PredictScalar(Span destination, int destinationStride, ReadOnlySpan above, ReadOnlySpan left, int width, int height); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PaethPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PaethPredictor.cs deleted file mode 100644 index 025d6256e3..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PaethPredictor.cs +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block by selecting the top, left, or top-left neighbor with the smallest local gradient. -/// -internal readonly struct Av1PaethPredictor : IAv1Predictor -{ - /// - /// The number of top samples consumed and samples written to each destination row. - /// - private readonly uint blockWidth; - - /// - /// The number of left samples consumed and destination rows written. - /// - private readonly uint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1PaethPredictor(Size blockSize) - { - this.blockWidth = (uint)blockSize.Width; - this.blockHeight = (uint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1PaethPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (uint)transformSize.GetWidth(); - this.blockHeight = (uint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block using the Paeth gradient selector. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples, preceded in memory by the top-left sample. - /// The left neighboring samples. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1PaethPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte aboveRef = ref above[0]; - - // The caller preserves the top-left sample immediately before the top row. - int yTopLeft = Unsafe.Subtract(ref aboveRef, 1); - ref byte destinationRef = ref destination[0]; - for (nuint r = 0; r < this.blockHeight; r++) - { - for (nuint c = 0; c < this.blockWidth; c++) - { - Unsafe.Add(ref destinationRef, c) = PredictSingle( - Unsafe.Add(ref leftRef, r), - Unsafe.Add(ref aboveRef, c), - yTopLeft); - } - - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } - - /// - /// Selects the reference sample nearest to the planar estimate + - . - /// - /// The left reference sample. - /// The top reference sample. - /// The shared top-left reference sample. - /// The reference sample with the smallest absolute distance from the planar estimate. - private static byte PredictSingle(byte left, byte top, int topLeft) - { - int basis = top + left - topLeft; - int pLeft = Av1Math.AbsoluteDifference(basis, left); - int pTop = Av1Math.AbsoluteDifference(basis, top); - int pTopLeft = Av1Math.AbsoluteDifference(basis, topLeft); - - // Ties prefer left, then top, matching AV1's normative Paeth selection order. - return (byte)((pLeft <= pTop && pLeft <= pTopLeft) ? left : (pTop <= pTopLeft) ? top : topLeft); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs index 68a38f8a92..2b23d75094 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs @@ -4,6 +4,8 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -24,6 +26,21 @@ internal class Av1PredictionDecoder /// private const int MaxUpsampleSize = 16; + /// + /// The number of samples reserved for one prepared AV1 intra-prediction edge. + /// + private const int ReferenceBufferLength = (Av1Constants.MaxTransformSize * 2) + 32; + + /// + /// The padded sample count required by the widest intra-edge SIMD loads. + /// + private const int EdgeScratchLength = 160; + + /// + /// The number of high-bit-depth samples required by the reusable prediction workspace. + /// + public const int ScratchLength = Av1DirectionalIntraPredictor.ScratchLength + (2 * ReferenceBufferLength) + EdgeScratchLength; + /// /// The sequence-level syntax that controls chroma sampling, bit depth, superblock size, and intra-edge filtering. /// @@ -34,15 +51,22 @@ internal class Av1PredictionDecoder /// private readonly ObuFrameHeader frameHeader; + /// + /// The frame-owned workspace shared by directional and filter-intra predictors. + /// + private readonly Memory predictorScratch; + /// /// Initializes a new instance of the class. /// /// The decoded sequence header for the current image. /// The decoded frame header for the current image. - public Av1PredictionDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + /// The reusable predictor workspace owned by the containing block decoder. + public Av1PredictionDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, Memory predictorScratch) { this.sequenceHeader = sequenceHeader; this.frameHeader = frameHeader; + this.predictorScratch = predictorScratch; } /// @@ -791,10 +815,15 @@ internal class Av1PredictionDecoder { int baseValue = 128 << (bitDepth - 8); + // The frame-owned allocation is sized in high-bit-depth samples. Reinterpreting it as T gives the byte + // path additional capacity while preserving the same sample offsets for the larger short representation. + Span scratch = MemoryMarshal.Cast(this.predictorScratch.Span); + Span aboveData = scratch.Slice(Av1DirectionalIntraPredictor.ScratchLength, ReferenceBufferLength); + Span leftData = scratch.Slice(Av1DirectionalIntraPredictor.ScratchLength + ReferenceBufferLength, ReferenceBufferLength); + Span edgeScratch = scratch.Slice(Av1DirectionalIntraPredictor.ScratchLength + (2 * ReferenceBufferLength), EdgeScratchLength); + // Prefix storage is required because AV1 addresses the shared top-left sample at -1 // and writes upsampled edge samples as far back as -2. - Span aboveData = stackalloc T[(Av1Constants.MaxTransformSize * 2) + 32]; - Span leftData = stackalloc T[(Av1Constants.MaxTransformSize * 2) + 32]; aboveData.Fill(T.CreateChecked(baseValue - 1)); leftData.Fill(T.CreateChecked(baseValue + 1)); Span aboveRow = aboveData[16..]; @@ -989,7 +1018,7 @@ internal class Av1PredictionDecoder if (useFilterIntra) { - FilterIntraPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode, bitDepth); + this.FilterIntraPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode, bitDepth); return; } @@ -1016,14 +1045,14 @@ internal class Av1PredictionDecoder { int strength = IntraEdgeFilterStrength(transformWidth, transformHeight, angle - 90, filterType); int pixelCount = topPixelCount + ab_le + (needRight ? transformHeight : 0); - FilterIntraEdge(ref Unsafe.Subtract(ref aboveRow[0], ab_le), pixelCount, strength); + FilterIntraEdge(ref Unsafe.Subtract(ref aboveRow[0], ab_le), pixelCount, strength, edgeScratch); } if (needLeft && leftPixelCount > 0) { int strength = IntraEdgeFilterStrength(transformHeight, transformWidth, angle - 180, filterType); int pixelCount = leftPixelCount + ab_le + (needBottom ? transformWidth : 0); - FilterIntraEdge(ref Unsafe.Subtract(ref leftColumn[0], ab_le), pixelCount, strength); + FilterIntraEdge(ref Unsafe.Subtract(ref leftColumn[0], ab_le), pixelCount, strength, edgeScratch); } } @@ -1032,7 +1061,7 @@ internal class Av1PredictionDecoder { int pixelCount = transformWidth + (needRight ? transformHeight : 0); - UpsampleIntraEdge(aboveRow, pixelCount, bitDepth); + UpsampleIntraEdge(aboveRow, pixelCount, bitDepth, edgeScratch); } upsampleLeft = UseIntraEdgeUpsample(transformHeight, transformWidth, angle - 180, filterType); @@ -1040,11 +1069,11 @@ internal class Av1PredictionDecoder { int pixelCount = transformHeight + (needBottom ? transformWidth : 0); - UpsampleIntraEdge(leftColumn, pixelCount, bitDepth); + UpsampleIntraEdge(leftColumn, pixelCount, bitDepth, edgeScratch); } } - DirectionalPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle, bitDepth); + this.DirectionalPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle); return; } @@ -1073,29 +1102,34 @@ internal class Av1PredictionDecoder private static void DcPredictor(bool hasLeft, bool hasAbove, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span above, Span left, int bitDepth) where T : unmanaged { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + // DecodeCore is reachable only through byte and short overloads, so this type // dispatch permits shared reference preparation without boxing or allocating. if (typeof(T) == typeof(byte)) { - Av1PredictorFactory.DcPredictor( + Av1DcIntraPredictor.Predict( hasLeft, hasAbove, - transformSize, MemoryMarshal.Cast(destination), - destinationStride, + (int)destinationStride, MemoryMarshal.Cast(above), - MemoryMarshal.Cast(left)); + MemoryMarshal.Cast(left), + width, + height); } else { - Av1PredictorFactory.DcPredictor( + Av1DcIntraPredictor.Predict( hasLeft, hasAbove, - transformSize, MemoryMarshal.Cast(destination), - destinationStride, + (int)destinationStride, MemoryMarshal.Cast(above), MemoryMarshal.Cast(left), + width, + height, bitDepth); } } @@ -1113,25 +1147,29 @@ internal class Av1PredictionDecoder private static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span above, Span left) where T : unmanaged { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + Av1IntraPredictorBase predictor = Av1IntraPredictorBase.GetPredictor(mode); + if (typeof(T) == typeof(byte)) { - Av1PredictorFactory.GeneralPredictor( - mode, - transformSize, + predictor.Predict( MemoryMarshal.Cast(destination), - destinationStride, + (int)destinationStride, MemoryMarshal.Cast(above), - MemoryMarshal.Cast(left)); + MemoryMarshal.Cast(left), + width, + height); } else { - Av1PredictorFactory.GeneralPredictor( - mode, - transformSize, + predictor.Predict( MemoryMarshal.Cast(destination), - destinationStride, + (int)destinationStride, MemoryMarshal.Cast(above), - MemoryMarshal.Cast(left)); + MemoryMarshal.Cast(left), + width, + height); } } @@ -1147,34 +1185,35 @@ internal class Av1PredictionDecoder /// A value indicating whether the top edge was upsampled. /// A value indicating whether the left edge was upsampled. /// The adjusted prediction angle in degrees. - /// The number of bits used to represent each sample. - private static void DirectionalPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span above, Span left, bool upsampleAbove, bool upsampleLeft, int angle, int bitDepth) + private void DirectionalPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span above, Span left, bool upsampleAbove, bool upsampleLeft, int angle) where T : unmanaged { if (typeof(T) == typeof(byte)) { - Av1PredictorFactory.DirectionalPredictor( + Span scratch = MemoryMarshal.AsBytes(this.predictorScratch.Span)[..Av1DirectionalIntraPredictor.ScratchLength]; + Av1DirectionalIntraPredictor.Predict( MemoryMarshal.Cast(destination), - destinationStride, + (int)destinationStride, transformSize, MemoryMarshal.Cast(above), MemoryMarshal.Cast(left), upsampleAbove, upsampleLeft, - angle); + angle, + scratch); } else { - Av1PredictorFactory.DirectionalPredictor( + Av1DirectionalIntraPredictor.Predict( MemoryMarshal.Cast(destination), - destinationStride, + (int)destinationStride, transformSize, MemoryMarshal.Cast(above), MemoryMarshal.Cast(left), upsampleAbove, upsampleLeft, angle, - bitDepth); + this.predictorScratch.Span); } } @@ -1189,29 +1228,36 @@ internal class Av1PredictionDecoder /// The prepared left reference samples. /// The filter intra mode whose coefficient set is applied. /// The number of bits used to represent each sample. - private static void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span above, Span left, Av1FilterIntraMode mode, int bitDepth) + private void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span above, Span left, Av1FilterIntraMode mode, int bitDepth) where T : unmanaged { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + Av1FilterIntraPredictorBase predictor = Av1FilterIntraPredictorBase.GetPredictor(mode); + if (typeof(T) == typeof(byte)) { - Av1PredictorFactory.FilterIntraPredictor( + Span scratch = MemoryMarshal.AsBytes(this.predictorScratch.Span)[..Av1FilterIntraPredictorBase.ScratchLength]; + predictor.Predict( MemoryMarshal.Cast(destination), - destinationStride, - transformSize, + (int)destinationStride, MemoryMarshal.Cast(above), MemoryMarshal.Cast(left), - mode); + width, + height, + scratch); } else { - Av1PredictorFactory.FilterIntraPredictor( + predictor.Predict( MemoryMarshal.Cast(destination), - destinationStride, - transformSize, + (int)destinationStride, MemoryMarshal.Cast(above), MemoryMarshal.Cast(left), - mode, - bitDepth); + width, + height, + bitDepth, + this.predictorScratch.Span[..Av1FilterIntraPredictorBase.ScratchLength]); } } @@ -1222,36 +1268,199 @@ internal class Av1PredictionDecoder /// The edge buffer, including writable prefix storage at indices -2 and -1. /// The number of original edge samples to upsample. /// The number of bits used to clamp interpolated samples. - private static void UpsampleIntraEdge(Span buffer, int count, int bitDepth) + /// The reusable padded source workspace. + private static void UpsampleIntraEdge(Span buffer, int count, int bitDepth, Span scratch) where T : unmanaged, IBinaryInteger { - // TODO: Consider creating SIMD version DebugGuard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count)); - Span input = stackalloc T[MaxUpsampleSize + 3]; - T beforeBuffer = Unsafe.Subtract(ref buffer[0], 1); + // DecodeBuildIntraPredictors is closed only over byte and short. Keeping that dispatch outside the + // kernels gives the JIT concrete vector element types and removes generic arithmetic from their loops. + if (typeof(T) == typeof(byte)) + { + UpsampleIntraEdge(MemoryMarshal.Cast(buffer), count, MemoryMarshal.Cast(scratch)); + } + else + { + UpsampleIntraEdge(MemoryMarshal.Cast(buffer), count, bitDepth, MemoryMarshal.Cast(scratch)); + } + } + + /// + /// Inserts half-sample positions into an 8-bit intra-prediction edge. + /// + /// The edge buffer, including writable prefix storage at indices -2 and -1. + /// The number of original edge samples to upsample. + /// The reusable padded source workspace. + public static void UpsampleIntraEdge(Span buffer, int count, Span scratch) + { + ref byte bufferBase = ref MemoryMarshal.GetReference(buffer); + ref byte inputBase = ref MemoryMarshal.GetReference(scratch); + byte beforeBuffer = Unsafe.Subtract(ref bufferBase, 1); + byte finalSample = Unsafe.Add(ref bufferBase, count - 1); + + // Vector loads intentionally extend past the logical edge. Initializing the complete load window with + // the final sample provides the AV1 endpoint extension and keeps every unaligned read inside scratch. + scratch[..32].Fill(finalSample); + inputBase = beforeBuffer; + Unsafe.Add(ref inputBase, 1) = beforeBuffer; + buffer[..count].CopyTo(scratch[2..]); + Unsafe.Subtract(ref bufferBase, 2) = beforeBuffer; + + int i = 0; + if (Vector128.IsHardwareAccelerated) + { + int eightSamplesFromEnd = count - 8; + for (; i <= eightSamplesFromEnd; i += 8) + { + Vector128 interpolated = InterpolateEightBytes(ref inputBase, i); + Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); + Vector128 interleaved = Vector128_.UnpackLow(interpolated, originals); + interleaved.StoreUnsafe(ref Unsafe.Add(ref bufferBase, (2 * i) - 1)); + } + + // AV1 upsampled edges are normally multiples of four. A half-vector store handles that common + // remainder without overwriting the prepared extension beyond the logical output edge. + if (i <= count - 4) + { + Vector128 interpolated = InterpolateEightBytes(ref inputBase, i); + Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); + Vector128 interleaved = Vector128_.UnpackLow(interpolated, originals); + Unsafe.As(ref Unsafe.Add(ref bufferBase, (2 * i) - 1)) = interleaved.AsUInt64().GetElement(0); + i += 4; + } + } - // Duplicate both endpoints so the four-tap interpolation kernel can run at - // the edge without a separate boundary branch for its outer samples. - input[0] = beforeBuffer; - input[1] = beforeBuffer; - for (int i = 0; i < count; i++) + UpsampleIntraEdgeScalar(ref bufferBase, ref inputBase, i, count, 255); + } + + /// + /// Inserts half-sample positions into a high-bit-depth intra-prediction edge. + /// + /// The edge buffer, including writable prefix storage at indices -2 and -1. + /// The number of original edge samples to upsample. + /// The number of bits used to clamp interpolated samples. + /// The reusable padded source workspace. + public static void UpsampleIntraEdge(Span buffer, int count, int bitDepth, Span scratch) + { + ref short bufferBase = ref MemoryMarshal.GetReference(buffer); + ref short inputBase = ref MemoryMarshal.GetReference(scratch); + short beforeBuffer = Unsafe.Subtract(ref bufferBase, 1); + short finalSample = Unsafe.Add(ref bufferBase, count - 1); + + // The same padded layout is used for 10- and 12-bit edges. Arithmetic widens to Int32 before applying + // the four-tap kernel because the 12-bit intermediate exceeds the unsigned 16-bit range. + scratch[..32].Fill(finalSample); + inputBase = beforeBuffer; + Unsafe.Add(ref inputBase, 1) = beforeBuffer; + buffer[..count].CopyTo(scratch[2..]); + Unsafe.Subtract(ref bufferBase, 2) = beforeBuffer; + + int maximum = (1 << bitDepth) - 1; + int i = 0; + if (Vector128.IsHardwareAccelerated) { - input[i + 2] = buffer[i]; + int eightSamplesFromEnd = count - 8; + for (; i <= eightSamplesFromEnd; i += 8) + { + Vector128 interpolated = InterpolateEightHighBitDepthSamples(ref inputBase, i, maximum); + Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); + Vector128 interleavedLow = Vector128_.UnpackLow(interpolated, originals); + Vector128 interleavedHigh = Vector128_.UnpackHigh(interpolated, originals); + ref short destination = ref Unsafe.Add(ref bufferBase, (2 * i) - 1); + + interleavedLow.StoreUnsafe(ref destination); + interleavedHigh.StoreUnsafe(ref destination, (nuint)Vector128.Count); + } + + if (i <= count - 4) + { + Vector128 interpolated = InterpolateEightHighBitDepthSamples(ref inputBase, i, maximum); + Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); + Vector128 interleaved = Vector128_.UnpackLow(interpolated, originals); + interleaved.StoreUnsafe(ref Unsafe.Add(ref bufferBase, (2 * i) - 1)); + i += 4; + } } - input[count + 2] = buffer[count - 1]; + UpsampleIntraEdgeScalar(ref bufferBase, ref inputBase, i, count, maximum); + } + + /// + /// Calculates eight 8-bit half-sample values in parallel. + /// + /// The first padded input sample. + /// The first output sample index. + /// The interpolated samples in the lower eight lanes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 InterpolateEightBytes(ref byte input, int offset) + { + Vector128 source0 = Vector128.LoadUnsafe(ref input, (nuint)offset); + Vector128 source1 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 1)); + Vector128 source2 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 2)); + Vector128 source3 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 3)); + (Vector128 source0Low, _) = Vector128.Widen(source0); + (Vector128 source1Low, _) = Vector128.Widen(source1); + (Vector128 source2Low, _) = Vector128.Widen(source2); + (Vector128 source3Low, _) = Vector128.Widen(source3); + Vector128 interpolation = (((source1Low + source2Low) * Vector128.Create((ushort)9)) - (source0Low + source3Low)).AsInt16(); + + interpolation = Vector128.Clamp((interpolation + Vector128.Create((short)8)) >> 4, Vector128.Zero, Vector128.Create((short)255)); + return Vector128.Narrow(interpolation.AsUInt16(), Vector128.Zero); + } + + /// + /// Calculates eight high-bit-depth half-sample values in parallel. + /// + /// The first padded input sample. + /// The first output sample index. + /// The maximum reconstructed sample value. + /// The interpolated samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 InterpolateEightHighBitDepthSamples(ref short input, int offset, int maximum) + { + Vector128 source0 = Vector128.LoadUnsafe(ref input, (nuint)offset); + Vector128 source1 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 1)); + Vector128 source2 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 2)); + Vector128 source3 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 3)); + (Vector128 source0Low, Vector128 source0High) = Vector128.Widen(source0); + (Vector128 source1Low, Vector128 source1High) = Vector128.Widen(source1); + (Vector128 source2Low, Vector128 source2High) = Vector128.Widen(source2); + (Vector128 source3Low, Vector128 source3High) = Vector128.Widen(source3); + Vector128 coefficient = Vector128.Create(9); + Vector128 rounding = Vector128.Create(8); + Vector128 maximumVector = Vector128.Create(maximum); + Vector128 low = ((((source1Low + source2Low) * coefficient) - (source0Low + source3Low)) + rounding) >> 4; + Vector128 high = ((((source1High + source2High) * coefficient) - (source0High + source3High)) + rounding) >> 4; + + low = Vector128.Clamp(low, Vector128.Zero, maximumVector); + high = Vector128.Clamp(high, Vector128.Zero, maximumVector); + return Vector128.Narrow(low, high); + } - Unsafe.Subtract(ref buffer[0], 2) = input[0]; - ref T output = ref buffer[0]; - for (int i = 0; i < count; i++) + /// + /// Inserts the scalar remainder of an intra-edge upsample operation. + /// + /// The byte or 16-bit sample type. + /// The first original edge sample. + /// The first padded input sample. + /// The first sample not processed by SIMD. + /// The number of original edge samples. + /// The maximum reconstructed sample value. + private static void UpsampleIntraEdgeScalar(ref T buffer, ref T input, int start, int count, int maximum) + where T : unmanaged, IBinaryInteger + { + for (int i = start; i < count; i++) { - int s = -int.CreateChecked(input[i]) + (9 * int.CreateChecked(input[i + 1])) + (9 * int.CreateChecked(input[i + 2])) - int.CreateChecked(input[i + 3]); - s = Av1Math.Clamp((s + 8) >> 4, 0, (1 << bitDepth) - 1); + int interpolation = -int.CreateChecked(Unsafe.Add(ref input, i)) + + (9 * int.CreateChecked(Unsafe.Add(ref input, i + 1))) + + (9 * int.CreateChecked(Unsafe.Add(ref input, i + 2))) + - int.CreateChecked(Unsafe.Add(ref input, i + 3)); + + interpolation = Av1Math.Clamp((interpolation + 8) >> 4, 0, maximum); - // The AOM edge buffer reserves prefix storage for the samples at indices -2 and -1. - Unsafe.Add(ref output, (2 * i) - 1) = T.CreateChecked(s); - Unsafe.Add(ref output, 2 * i) = input[i + 2]; + Unsafe.Add(ref buffer, (2 * i) - 1) = T.CreateChecked(interpolation); + Unsafe.Add(ref buffer, 2 * i) = Unsafe.Add(ref input, i + 2); } } @@ -1283,38 +1492,261 @@ internal class Av1PredictionDecoder /// A reference to the first edge sample to filter. /// The number of edge samples. /// The AV1 filter-strength index from zero through three. + /// The reusable padded source workspace. /// Corresponds to svt_av1_filter_intra_edge_c in SVT-AV1. - private static void FilterIntraEdge(ref T buffer, int count, int strength) + private static void FilterIntraEdge(ref T buffer, int count, int strength, Span scratch) where T : unmanaged, IBinaryInteger { - // TODO: Consider creating SIMD version if (strength == 0) { return; } - int[][] kernel = [ - [0, 4, 8, 4, 0], [0, 5, 6, 5, 0], [2, 4, 4, 4, 2] - ]; - int filt = strength - 1; - Span edge = stackalloc T[129]; + // As with edge upsampling, closing the kernel over a concrete sample type keeps vector arithmetic + // outside the generic decoder while the valid strength-zero no-op remains at the owning boundary. + if (typeof(T) == typeof(byte)) + { + FilterIntraEdge(ref Unsafe.As(ref buffer), count, strength, MemoryMarshal.Cast(scratch)); + } + else + { + FilterIntraEdge(ref Unsafe.As(ref buffer), count, strength, MemoryMarshal.Cast(scratch)); + } + } - // Filtering must read the original edge throughout the convolution, so retain - // a scratch copy rather than feeding earlier filtered samples into later outputs. - MemoryMarshal.CreateSpan(ref buffer, count).CopyTo(edge); - for (int i = 1; i < count; i++) + /// + /// Applies an AV1 intra-edge smoothing kernel to 8-bit samples. + /// + /// A reference to the first edge sample to filter. + /// The number of edge samples. + /// The AV1 filter-strength index from one through three. + /// The reusable padded source workspace. + public static void FilterIntraEdge(ref byte buffer, int count, int strength, Span scratch) + { + byte finalSample = Unsafe.Add(ref buffer, count - 1); + + // The original edge is retained because each convolution window must observe unfiltered neighbors. + // Padding both endpoints also makes every complete vector use the same contiguous load pattern. + scratch[..EdgeScratchLength].Fill(finalSample); + scratch[0] = buffer; + MemoryMarshal.CreateReadOnlySpan(ref buffer, count).CopyTo(scratch[1..]); + + ref byte edge = ref MemoryMarshal.GetReference(scratch); + int outputCount = count - 1; + int processed = 0; + if (Vector128.IsHardwareAccelerated) { - int s = 0; - for (int j = 0; j < 5; j++) + int eightSamplesFromEnd = outputCount - 8; + switch (strength) { - int k = i - 2 + j; - k = (k < 0) ? 0 : k; - k = (k > count - 1) ? count - 1 : k; - s += int.CreateChecked(edge[k]) * kernel[filt][j]; + case 1: + for (; processed <= eightSamplesFromEnd; processed += 8) + { + Vector128 source0 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1))); + Vector128 source1 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2))); + Vector128 source2 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3))); + Vector128 result = Vector128.Narrow(FilterEdgeStrength1(source0, source1, source2), Vector128.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, processed + 1)) = result.AsUInt64().GetElement(0); + } + + break; + case 2: + for (; processed <= eightSamplesFromEnd; processed += 8) + { + Vector128 source0 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1))); + Vector128 source1 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2))); + Vector128 source2 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3))); + Vector128 result = Vector128.Narrow(FilterEdgeStrength2(source0, source1, source2), Vector128.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, processed + 1)) = result.AsUInt64().GetElement(0); + } + + break; + default: + for (; processed <= eightSamplesFromEnd; processed += 8) + { + Vector128 source0 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)processed)); + Vector128 source1 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1))); + Vector128 source2 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2))); + Vector128 source3 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3))); + Vector128 source4 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 4))); + Vector128 result = Vector128.Narrow(FilterEdgeStrength3(source0, source1, source2, source3, source4), Vector128.Zero); + Unsafe.As(ref Unsafe.Add(ref buffer, processed + 1)) = result.AsUInt64().GetElement(0); + } + + break; } + } + + FilterIntraEdgeScalar(ref buffer, ref edge, processed, outputCount, strength); + } - s = (s + 8) >> 4; - Unsafe.Add(ref buffer, i) = T.CreateChecked(s); + /// + /// Applies an AV1 intra-edge smoothing kernel to high-bit-depth samples. + /// + /// A reference to the first edge sample to filter. + /// The number of edge samples. + /// The AV1 filter-strength index from one through three. + /// The reusable padded source workspace. + public static void FilterIntraEdge(ref short buffer, int count, int strength, Span scratch) + { + short finalSample = Unsafe.Add(ref buffer, count - 1); + + scratch[..EdgeScratchLength].Fill(finalSample); + scratch[0] = buffer; + MemoryMarshal.CreateReadOnlySpan(ref buffer, count).CopyTo(scratch[1..]); + + ref short edge = ref MemoryMarshal.GetReference(scratch); + int outputCount = count - 1; + int processed = 0; + + // The largest 12-bit weighted sum is 65520; the greatest rounding bias raises that only to 65528. + // Unsigned 16-bit lanes therefore preserve every normative strength without widening to 32-bit vectors. + if (Vector128.IsHardwareAccelerated) + { + int eightSamplesFromEnd = outputCount - Vector128.Count; + switch (strength) + { + case 1: + for (; processed <= eightSamplesFromEnd; processed += Vector128.Count) + { + Vector128 source0 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1)).AsUInt16(); + Vector128 source1 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2)).AsUInt16(); + Vector128 source2 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3)).AsUInt16(); + FilterEdgeStrength1(source0, source1, source2).AsInt16().StoreUnsafe(ref buffer, (nuint)(processed + 1)); + } + + break; + case 2: + for (; processed <= eightSamplesFromEnd; processed += Vector128.Count) + { + Vector128 source0 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1)).AsUInt16(); + Vector128 source1 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2)).AsUInt16(); + Vector128 source2 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3)).AsUInt16(); + FilterEdgeStrength2(source0, source1, source2).AsInt16().StoreUnsafe(ref buffer, (nuint)(processed + 1)); + } + + break; + default: + for (; processed <= eightSamplesFromEnd; processed += Vector128.Count) + { + Vector128 source0 = Vector128.LoadUnsafe(ref edge, (nuint)processed).AsUInt16(); + Vector128 source1 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1)).AsUInt16(); + Vector128 source2 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2)).AsUInt16(); + Vector128 source3 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3)).AsUInt16(); + Vector128 source4 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 4)).AsUInt16(); + FilterEdgeStrength3(source0, source1, source2, source3, source4).AsInt16().StoreUnsafe(ref buffer, (nuint)(processed + 1)); + } + + break; + } + } + + FilterIntraEdgeScalar(ref buffer, ref edge, processed, outputCount, strength); + } + + /// + /// Widens the lower eight lanes of a byte vector for edge-filter arithmetic. + /// + /// The packed source samples. + /// The widened samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 WidenLower(Vector128 source) + { + (Vector128 lower, _) = Vector128.Widen(source); + return lower; + } + + /// + /// Applies the strength-one three-tap edge filter to eight samples. + /// + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The filtered samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 FilterEdgeStrength1(Vector128 source0, Vector128 source1, Vector128 source2) + => (source0 + (source1 << 1) + source2 + Vector128.Create((ushort)2)) >> 2; + + /// + /// Applies the strength-two three-tap edge filter to eight samples. + /// + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The filtered samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 FilterEdgeStrength2(Vector128 source0, Vector128 source1, Vector128 source2) + => (((source0 + source2) * Vector128.Create((ushort)5)) + (source1 * Vector128.Create((ushort)6)) + Vector128.Create((ushort)8)) >> 4; + + /// + /// Applies the strength-three five-tap edge filter to eight samples. + /// + /// The samples two positions before each output. + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The samples two positions after each output. + /// The filtered samples. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 FilterEdgeStrength3( + Vector128 source0, + Vector128 source1, + Vector128 source2, + Vector128 source3, + Vector128 source4) + => (source0 + ((source1 + source2 + source3) << 1) + source4 + Vector128.Create((ushort)4)) >> 3; + + /// + /// Applies an AV1 edge filter to samples not consumed by the vector loop. + /// + /// The byte or 16-bit sample type. + /// The first destination sample. + /// The first padded source sample. + /// The first output index not processed by SIMD. + /// The number of filtered outputs following the preserved first sample. + /// The AV1 filter-strength index. + private static void FilterIntraEdgeScalar(ref T buffer, ref T edge, int start, int count, int strength) + where T : unmanaged, IBinaryInteger + { + switch (strength) + { + case 1: + for (int i = start; i < count; i++) + { + int sourceOffset = i + 1; + int value = int.CreateChecked(Unsafe.Add(ref edge, sourceOffset)) + + (2 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 1))) + + int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 2)); + + Unsafe.Add(ref buffer, i + 1) = T.CreateChecked((value + 2) >> 2); + } + + break; + case 2: + for (int i = start; i < count; i++) + { + int sourceOffset = i + 1; + int value = (5 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset))) + + (6 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 1))) + + (5 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 2))); + + Unsafe.Add(ref buffer, i + 1) = T.CreateChecked((value + 8) >> 4); + } + + break; + default: + for (int i = start; i < count; i++) + { + int value = int.CreateChecked(Unsafe.Add(ref edge, i)) + + (2 * (int.CreateChecked(Unsafe.Add(ref edge, i + 1)) + + int.CreateChecked(Unsafe.Add(ref edge, i + 2)) + + int.CreateChecked(Unsafe.Add(ref edge, i + 3)))) + + int.CreateChecked(Unsafe.Add(ref edge, i + 4)); + + Unsafe.Add(ref buffer, i + 1) = T.CreateChecked((value + 4) >> 3); + } + + break; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs deleted file mode 100644 index c64d99c1d9..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Selects the scalar 8-bit or high-bit-depth AV1 intra predictor for a decoded prediction mode. -/// -internal class Av1PredictorFactory -{ - /// - /// The Q8 directional derivatives indexed by acute angle in degrees; zero entries represent angles AV1 does not signal. - /// - private static readonly int[] DirectionalIntraDerivative = [ - - // More evenly spread out angles and limited to 10-bit - // Values that are 0 will never be used - // Approx angle - 0, 0, 0, // 0 - 1023, 0, 0, // 3, ... - 547, 0, 0, // 6, ... - 372, 0, 0, 0, 0, // 9, ... - 273, 0, 0, // 14, ... - 215, 0, 0, // 17, ... - 178, 0, 0, // 20, ... - 151, 0, 0, // 23, ... (113 & 203 are base angles) - 132, 0, 0, // 26, ... - 116, 0, 0, // 29, ... - 102, 0, 0, 0, // 32, ... - 90, 0, 0, // 36, ... - 80, 0, 0, // 39, ... - 71, 0, 0, // 42, ... - 64, 0, 0, // 45, ... (45 & 135 are base angles) - 57, 0, 0, // 48, ... - 51, 0, 0, // 51, ... - 45, 0, 0, 0, // 54, ... - 40, 0, 0, // 58, ... - 35, 0, 0, // 61, ... - 31, 0, 0, // 64, ... - 27, 0, 0, // 67, ... (67 & 157 are base angles) - 23, 0, 0, // 70, ... - 19, 0, 0, // 73, ... - 15, 0, 0, 0, 0, // 76, ... - 11, 0, 0, // 81, ... - 7, 0, 0, // 84, ... - 3, 0, 0, // 87, ... - ]; - - /// - /// Predicts an 8-bit block from the average of whichever top and left neighbor edges are available. - /// - /// Whether the left neighboring column is available. - /// Whether the top neighboring row is available. - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples. - public static void DcPredictor(bool hasLeft, bool hasAbove, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span aboveRow, Span leftColumn) - { - if (hasLeft) - { - if (hasAbove) - { - Av1DcPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - } - else - { - Av1DcLeftPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - } - } - else - { - if (hasAbove) - { - Av1DcTopPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - } - else - { - Av1DcFillPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - } - } - } - - /// - /// Predicts a high-bit-depth block from the average of whichever top and left neighbor edges are available. - /// - /// Whether the left neighboring column is available. - /// Whether the top neighboring row is available. - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples. - /// The coded sample bit depth used to select the midpoint when no edge is available. - public static void DcPredictor(bool hasLeft, bool hasAbove, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span aboveRow, Span leftColumn, int bitDepth) - => Av1HighBitDepthPredictor.DcPredictor(hasLeft, hasAbove, transformSize, destination, destinationStride, aboveRow, leftColumn, bitDepth); - - /// - /// Predicts an 8-bit block by projecting reference-edge samples along a directional angle. - /// - /// The destination block. - /// The distance, in samples, between destination rows. - /// The predicted block dimensions. - /// The top and top-right reference samples. - /// The left and bottom-left reference samples. - /// Whether the top reference edge is stored at half-sample intervals. - /// Whether the left reference edge is stored at half-sample intervals. - /// The prediction angle in degrees from 1 through 269. - /// SVT-AV1: svt_aom_highbd_dr_predictor. - public static void DirectionalPredictor(Span destination, nuint stride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, bool upsampleAbove, bool upsampleLeft, int angle) - { - int dx = GetDeltaX(angle); - int dy = GetDeltaY(angle); - int bw = transformSize.GetWidth(); - int bh = transformSize.GetHeight(); - Guard.MustBeBetweenOrEqualTo(angle, 1, 269, nameof(angle)); - - // The three open quadrants select which reference edge or edge pair the projected ray intersects. - // Exact 90- and 180-degree modes reduce to copying the top row or left column without interpolation. - if (angle is > 0 and < 90) - { - Av1DirectionalZone1Predictor.PredictScalar(transformSize, destination, stride, aboveRow, upsampleAbove, dx); - } - else if (angle is > 90 and < 180) - { - Av1DirectionalZone2Predictor.PredictScalar(transformSize, destination, stride, aboveRow, leftColumn, upsampleAbove, upsampleLeft, dx, dy); - } - else if (angle is > 180 and < 270) - { - Av1DirectionalZone3Predictor.PredictScalar(transformSize, destination, stride, leftColumn, upsampleLeft, dx, dy); - } - else if (angle == 90) - { - Av1VerticalPredictor.PredictScalar(transformSize, destination, stride, aboveRow, leftColumn); - } - else if (angle == 180) - { - Av1HorizontalPredictor.PredictScalar(transformSize, destination, stride, aboveRow, leftColumn); - } - } - - /// - /// Predicts a high-bit-depth block by projecting reference-edge samples along a directional angle. - /// - /// The destination block. - /// The distance, in samples, between destination rows. - /// The predicted block dimensions. - /// The top and top-right reference samples. - /// The left and bottom-left reference samples. - /// Whether the top reference edge is stored at half-sample intervals. - /// Whether the left reference edge is stored at half-sample intervals. - /// The prediction angle in degrees from 1 through 269. - /// The coded sample bit depth used to clamp interpolated values. - public static void DirectionalPredictor(Span destination, nuint stride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, bool upsampleAbove, bool upsampleLeft, int angle, int bitDepth) - => Av1HighBitDepthPredictor.DirectionalPredictor(destination, stride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle, bitDepth); - - /// - /// Predicts an 8-bit block using the selected AV1 filter-intra kernel. - /// - /// The destination block. - /// The distance, in samples, between destination rows. - /// The predicted block dimensions. - /// The top neighboring samples. - /// The left neighboring samples. - /// The filter-intra coefficient set. - public static void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, Av1FilterIntraMode filterIntraMode) - => Av1FilterIntraPredictor.Predict(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode); - - /// - /// Predicts a high-bit-depth block using the selected AV1 filter-intra kernel. - /// - /// The destination block. - /// The distance, in samples, between destination rows. - /// The predicted block dimensions. - /// The top neighboring samples. - /// The left neighboring samples. - /// The filter-intra coefficient set. - /// The coded sample bit depth used to clamp filtered values. - public static void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, Av1FilterIntraMode filterIntraMode, int bitDepth) - => Av1HighBitDepthPredictor.FilterIntraPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode, bitDepth); - - /// - /// Selects an 8-bit horizontal, vertical, Paeth, or smooth predictor for a non-directional mode. - /// - /// The non-directional prediction mode. - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples. - public static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span aboveRow, Span leftColumn) - { - switch (mode) - { - case Av1PredictionMode.Horizontal: - Av1HorizontalPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - break; - case Av1PredictionMode.Vertical: - Av1VerticalPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - break; - case Av1PredictionMode.Paeth: - Av1PaethPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - break; - case Av1PredictionMode.Smooth: - Av1SmoothPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - break; - case Av1PredictionMode.SmoothHorizontal: - Av1SmoothHorizontalPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - break; - case Av1PredictionMode.SmoothVertical: - Av1SmoothVerticalPredictor.PredictScalar(transformSize, destination, destinationStride, aboveRow, leftColumn); - break; - } - } - - /// - /// Selects a high-bit-depth horizontal, vertical, Paeth, or smooth predictor for a non-directional mode. - /// - /// The non-directional prediction mode. - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples. - public static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span aboveRow, Span leftColumn) - => Av1HighBitDepthPredictor.GeneralPredictor(mode, transformSize, destination, destinationStride, aboveRow, leftColumn); - - /// - /// Gets the Q8 vertical displacement per unit horizontal displacement for a directional angle. - /// - /// The prediction angle in degrees. - /// The Q8 vertical derivative, or one when the selected directional zone does not consume it. - public static int GetDeltaY(int angle) - { - if (angle is > 90 and < 180) - { - return DirectionalIntraDerivative[angle - 90]; - } - else if (angle is > 180 and < 270) - { - return DirectionalIntraDerivative[270 - angle]; - } - else - { - // Zones one and the exact horizontal/vertical modes never consume dy; one avoids a zero placeholder. - return 1; - } - } - - /// - /// Gets the Q8 horizontal displacement per unit vertical displacement for a directional angle. - /// - /// The prediction angle in degrees. - /// The Q8 horizontal derivative, or one when the selected directional zone does not consume it. - public static int GetDeltaX(int angle) - { - if (angle is > 0 and < 90) - { - return DirectionalIntraDerivative[angle]; - } - else if (angle is > 90 and < 180) - { - return DirectionalIntraDerivative[180 - angle]; - } - else - { - // Zone three and the exact horizontal/vertical modes never consume dx; one avoids a zero placeholder. - return 1; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothHorizontalPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothHorizontalPredictor.cs deleted file mode 100644 index 8384089672..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothHorizontalPredictor.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block by smoothly blending each left neighbor toward the top-right reference sample. -/// -internal readonly struct Av1SmoothHorizontalPredictor : IAv1Predictor -{ - /// - /// The number of interpolation weights and samples written to each destination row. - /// - private readonly nuint blockWidth; - - /// - /// The number of left samples consumed and destination rows written. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1SmoothHorizontalPredictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1SmoothHorizontalPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block using horizontal smooth interpolation. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples whose final value supplies the right endpoint. - /// The left neighboring samples. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1SmoothHorizontalPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - /// SVT-AV1: highbd_smooth_h_predictor. - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - int rightPrediction = Unsafe.Add(ref aboveRef, this.blockWidth - 1); // estimated by top-right pixel - ref int weights = ref Av1SmoothPredictor.Weights[(int)this.blockWidth]; - - int log2Scale = Av1SmoothPredictor.WeightLog2Scale; - int scale = 1 << Av1SmoothPredictor.WeightLog2Scale; - - // sm_weights_sanity_checks(sm_weights_w, sm_weights_h, scale, log2_scale + 2); - for (nuint r = 0; r < this.blockHeight; ++r) - { - for (nuint c = 0; c < this.blockWidth; ++c) - { - int columnWeight = Unsafe.Add(ref weights, c); - Guard.MustBeGreaterThanOrEqualTo(scale, columnWeight, nameof(scale)); - - // The Q8 weight decreases toward the right edge, shifting influence from left to top-right. - int thisPredition = Unsafe.Add(ref leftRef, r) * columnWeight; - thisPredition += rightPrediction * (scale - columnWeight); - Unsafe.Add(ref destinationRef, c) = (byte)Av1Math.DivideRound(thisPredition, log2Scale); - } - - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothPredictor.cs deleted file mode 100644 index 6fe36c316d..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothPredictor.cs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block by blending top-to-bottom and left-to-right smooth interpolation surfaces. -/// -internal readonly struct Av1SmoothPredictor : IAv1Predictor -{ - // Weights are quadratic from '1' to '1 / BlockSize', scaled by - // 2^sm_weight_log2_scale. - - /// - /// The number of fractional bits in the normative smooth-prediction weights. - /// - internal static readonly int WeightLog2Scale = 8; - - /// - /// The concatenated smooth-weight sequences, addressed by using the block dimension as the sequence offset. - /// - internal static readonly int[] Weights = [ - - // Unused, because we always offset by bs, which is at least 2. - 0, 0, - - // bs = 2 - 255, 128, - - // bs = 4 - 255, 149, 85, 64, - - // bs = 8 - 255, 197, 146, 105, 73, 50, 37, 32, - - // bs = 16 - 255, 225, 196, 170, 145, 123, 102, 84, 68, 54, 43, 33, 26, 20, 17, 16, - - // bs = 32 - 255, 240, 225, 210, 196, 182, 169, 157, 145, 133, 122, 111, 101, 92, 83, 74, - 66, 59, 52, 45, 39, 34, 29, 25, 21, 17, 14, 12, 10, 9, 8, 8, - - // bs = 64 - 255, 248, 240, 233, 225, 218, 210, 203, 196, 189, 182, 176, 169, 163, 156, - 150, 144, 138, 133, 127, 121, 116, 111, 106, 101, 96, 91, 86, 82, 77, 73, 69, - 65, 61, 57, 54, 50, 47, 44, 41, 38, 35, 32, 29, 27, 25, 22, 20, 18, 16, 15, - 13, 12, 10, 9, 8, 7, 6, 6, 5, 5, 4, 4, 4, - ]; - - /// - /// The number of top samples consumed and samples written to each destination row. - /// - private readonly nuint blockWidth; - - /// - /// The number of left samples consumed and destination rows written. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1SmoothPredictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1SmoothPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block by combining horizontal and vertical smooth interpolation. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1SmoothPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - /// SVT-AV1: highbd_smooth_predictor. - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - int belowPrediction = Unsafe.Add(ref leftRef, this.blockHeight - 1); // estimated by bottom-left pixel - int rightPrediction = Unsafe.Add(ref aboveRef, this.blockWidth - 1); // estimated by top-right pixel - ref int heightWeights = ref Weights[(int)this.blockHeight]; - ref int widthWeights = ref Weights[(int)this.blockWidth]; - - // The two independent interpolation surfaces each use Q8 weights, so their combined sum has one extra scale bit. - int log2Scale = 1 + WeightLog2Scale; - int scale = 1 << WeightLog2Scale; - - // sm_weights_sanity_checks(sm_weights_w, sm_weights_h, scale, log2_scale + 2); - for (nuint r = 0; r < this.blockHeight; ++r) - { - int rowWeight = Unsafe.Add(ref heightWeights, r); - Guard.MustBeGreaterThanOrEqualTo(scale, rowWeight, nameof(scale)); - for (nuint c = 0; c < this.blockWidth; ++c) - { - int columnWeight = Unsafe.Add(ref widthWeights, c); - Guard.MustBeGreaterThanOrEqualTo(scale, columnWeight, nameof(scale)); - - // Blend top toward bottom-left and left toward top-right, then normalize their combined Q8 contributions. - int thisPredition = Unsafe.Add(ref aboveRef, c) * rowWeight; - thisPredition += belowPrediction * (scale - rowWeight); - thisPredition += Unsafe.Add(ref leftRef, r) * columnWeight; - thisPredition += rightPrediction * (scale - columnWeight); - Unsafe.Add(ref destinationRef, c) = (byte)Av1Math.DivideRound(thisPredition, log2Scale); - } - - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothVerticalPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothVerticalPredictor.cs deleted file mode 100644 index dafff68296..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1SmoothVerticalPredictor.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block by smoothly blending each top neighbor toward the bottom-left reference sample. -/// -internal readonly struct Av1SmoothVerticalPredictor : IAv1Predictor -{ - /// - /// The number of top samples consumed and samples written to each destination row. - /// - private readonly nuint blockWidth; - - /// - /// The number of interpolation weights, left samples consumed, and destination rows written. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1SmoothVerticalPredictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1SmoothVerticalPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block using vertical smooth interpolation. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The left neighboring samples whose final value supplies the bottom endpoint. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1SmoothVerticalPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - /// SVT-AV1: highbd_smooth_v_predictor. - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte leftRef = ref left[0]; - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - int belowPrediction = Unsafe.Add(ref leftRef, this.blockHeight - 1); // estimated by bottom-left pixel - ref int weights = ref Av1SmoothPredictor.Weights[(int)this.blockHeight]; - - int log2Scale = Av1SmoothPredictor.WeightLog2Scale; - int scale = 1 << Av1SmoothPredictor.WeightLog2Scale; - - // sm_weights_sanity_checks(sm_weights_w, sm_weights_h, scale, log2_scale + 2); - for (nuint r = 0; r < this.blockHeight; ++r) - { - int rowWeight = Unsafe.Add(ref weights, r); - for (nuint c = 0; c < this.blockWidth; ++c) - { - // The Q8 weight decreases toward the bottom edge, shifting influence from top to bottom-left. - int thisPredition = Unsafe.Add(ref aboveRef, c) * rowWeight; - thisPredition += belowPrediction * (scale - rowWeight); - Unsafe.Add(ref destinationRef, c) = (byte)Av1Math.DivideRound(thisPredition, log2Scale); - } - - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1VerticalPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1VerticalPredictor.cs deleted file mode 100644 index 78a76cf384..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1VerticalPredictor.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Predicts an 8-bit AV1 block by copying the top neighboring samples into every destination row. -/// -internal readonly struct Av1VerticalPredictor : IAv1Predictor -{ - /// - /// The number of top samples consumed and samples written to each destination row. - /// - private readonly nuint blockWidth; - - /// - /// The number of destination rows. - /// - private readonly nuint blockHeight; - - /// - /// Initializes a new instance of the struct for explicit block dimensions. - /// - /// The predicted block dimensions in samples. - public Av1VerticalPredictor(Size blockSize) - { - this.blockWidth = (nuint)blockSize.Width; - this.blockHeight = (nuint)blockSize.Height; - } - - /// - /// Initializes a new instance of the struct for a transform size. - /// - /// The transform size whose dimensions define the predicted block. - public Av1VerticalPredictor(Av1TransformSize transformSize) - { - this.blockWidth = (nuint)transformSize.GetWidth(); - this.blockHeight = (nuint)transformSize.GetHeight(); - } - - /// - /// Predicts a transform block by extending its top edge vertically. - /// - /// The predicted block dimensions. - /// The destination block. - /// The distance, in samples, between destination rows. - /// The top neighboring samples. - /// The unused left-neighbor buffer required by the common predictor signature. - public static void PredictScalar(Av1TransformSize transformSize, Span destination, nuint stride, Span above, Span left) - => new Av1VerticalPredictor(transformSize).PredictScalar(destination, stride, above, left); - - /// - /// SVT-AV1: highbd_v_predictor. - public void PredictScalar(Span destination, nuint stride, Span above, Span left) - { - Guard.MustBeGreaterThanOrEqualTo(stride, this.blockWidth, nameof(stride)); - Guard.MustBeSizedAtLeast(left, (int)this.blockHeight, nameof(left)); - Guard.MustBeSizedAtLeast(above, (int)this.blockWidth, nameof(above)); - Guard.MustBeSizedAtLeast(destination, (int)this.blockHeight * (int)stride, nameof(destination)); - ref byte aboveRef = ref above[0]; - ref byte destinationRef = ref destination[0]; - uint width = (uint)this.blockWidth; - for (nuint r = 0; r < this.blockHeight; ++r) - { - Unsafe.CopyBlock(ref destinationRef, ref aboveRef, width); - destinationRef = ref Unsafe.Add(ref destinationRef, stride); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/IAv1Predictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/IAv1Predictor.cs deleted file mode 100644 index c9c4689982..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/IAv1Predictor.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Defines scalar reconstruction of an 8-bit AV1 intra-prediction block from its neighboring samples. -/// -internal interface IAv1Predictor -{ - /// - /// Writes the predicted block using scalar 8-bit arithmetic. - /// - /// The destination block, starting at its top-left sample. - /// The distance, in samples, between destination rows. - /// The neighboring samples immediately above the block. - /// The neighboring samples immediately left of the block. - public void PredictScalar(Span destination, nuint stride, Span above, Span left); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs index 4c443331f9..fc4450ca71 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs @@ -52,6 +52,16 @@ internal sealed class Av1BlockDecoder : IDisposable /// private readonly IMemoryOwner transformWorkspaceOwner; + /// + /// Owns the reusable directional and filter-intra prediction workspace. + /// + private readonly IMemoryOwner predictionScratchOwner; + + /// + /// Reconstructs intra-predicted blocks using the frame-owned prediction workspace. + /// + private readonly Av1PredictionDecoder predictionDecoder; + /// /// Indicates whether transform traversal must also populate loop-filter parameters. /// @@ -97,6 +107,8 @@ internal sealed class Av1BlockDecoder : IDisposable this.inverseQuantizationOwner = this.frameBuffer.MemoryAllocator.Allocate(inverseQuantizationSize); this.transformWorkspaceOwner = this.frameBuffer.MemoryAllocator.Allocate(Av1TransformWorkspace.MaximumLength); + this.predictionScratchOwner = this.frameBuffer.MemoryAllocator.Allocate(Av1PredictionDecoder.ScratchLength); + this.predictionDecoder = new(sequenceHeader, frameHeader, this.predictionScratchOwner.Memory); this.isLoopFilterEnabled = frameHeader.LoopFilterParameters.FilterLevel[0] != 0 || frameHeader.LoopFilterParameters.FilterLevel[1] != 0; @@ -114,6 +126,7 @@ internal sealed class Av1BlockDecoder : IDisposable /// public void Dispose() { + this.predictionScratchOwner.Dispose(); this.transformWorkspaceOwner.Dispose(); this.inverseQuantizationOwner.Dispose(); } @@ -184,7 +197,6 @@ internal sealed class Av1BlockDecoder : IDisposable : modeInfo.GetTransformUnitCount(Av1Plane.U); bool highBitDepth = this.frameBuffer.BytesPerSample == 2; - Av1PredictionDecoder predictionDecoder = new(this.sequenceHeader, this.frameHeader); for (int plane = 0; plane < colorConfig.PlaneCount; plane++) { int subX = (plane > 0) && colorConfig.SubSamplingX ? 1 : 0; @@ -277,7 +289,7 @@ internal sealed class Av1BlockDecoder : IDisposable // predicts its samples before any coded residual is added. if (highBitDepth) { - predictionDecoder.Decode( + this.predictionDecoder.Decode( partitionInfo, (Av1Plane)plane, transformSize, @@ -290,7 +302,7 @@ internal sealed class Av1BlockDecoder : IDisposable } else { - predictionDecoder.Decode( + this.predictionDecoder.Decode( partitionInfo, (Av1Plane)plane, transformSize, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs index 68573e6b46..26425679fa 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs @@ -1,640 +1,770 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.Formats.Heif.Av1; +using System.Numerics; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; -[Trait("Format", "Avif")] +/// +/// Verifies AV1 intra prediction against scalar definitions across the supported hardware-intrinsic configurations. +/// +[Trait("Format", "Heif")] public class Av1PredictorTests { - private static string[] Digests4x4 = [ - "7b1c762e28747f885d2b7d83cb8aa75c", "73353f179207f1432d40a132809e3a50", - "80c9237c838b0ec0674ccb070df633d5", "1cd79116b41fda884e7fa047f5eb14df", - "33211425772ee539a59981a2e9dc10c1", "d6f5f65a267f0e9a2752e8151cc1dcd7", - "7ff8c762cb766eb0665682152102ce4b", "2276b861ae4599de15938651961907ec", - "766982bc69f4aaaa8e71014c2dc219bc", "04401B397D702F853B12407EBFB91027", - ]; - - private static string[] Digests4x8 = [ - "0a0d8641ecfa0e82f541acdc894d5574", "1a40371af6cff9c278c5b0def9e4b3e7", - "3631a7a99569663b514f15b590523822", "646c7b592136285bd31501494e7393e7", - "ecbe89cc64dc2688123d3cfe865b5237", "79048e70ecbb7d43a4703f62718588c0", - "f3de11bf1198a00675d806d29c41d676", "32bb6cd018f6e871c342fcc21c7180cf", - "6f076a1e5ab3d69cf08811d62293e4be", "F94348B0D3E2B1F2FC061232831E8B9B", - ]; - - private static string[] Digests4x16 = [ - "cb8240be98444ede5ae98ca94afc1557", "460acbcf825a1fa0d8f2aa6bf2d6a21c", - "7896fdbbfe538dce1dc3a5b0873d74b0", "504aea29c6b27f21555d5516b8de2d8a", - "c5738e7fa82b91ea0e39232120da56ea", "19abbd934c243a6d9df7585d81332dd5", - "9e42b7b342e45c842dfa8aedaddbdfaa", "0e9eb07a89f8bf96bc219d5d1c3d9f6d", - "659393c31633e0f498bae384c9df5c7b", "812DFE5C38F2B837529C5A918756E8B8", - ]; - - private static string[] Digests8x4 = [ - "5950744064518f77867c8e14ebd8b5d7", "46b6cbdc76efd03f4ac77870d54739f7", - "efe21fd1b98cb1663950e0bf49483b3b", "3c647b64760b298092cbb8e2f5c06bfd", - "c3595929687ffb04c59b128d56e2632f", "d89ad2ddf8a74a520fdd1d7019fd75b4", - "53907cb70ad597ee5885f6c58201f98b", "09d2282a29008b7fb47eb60ed6653d06", - "e341fc1c910d7cb2dac5dbc58b9c9af9", "93D67CD49B07CE3870526E5C12F00EBF", - ]; - - private static string[] Digests8x8 = [ - "06fb7cb52719855a38b4883b4b241749", "2013aafd42a4303efb553e42264ab8b0", - "2f070511d5680c12ca73a20e47fd6e23", "9923705af63e454392625794d5459fe0", - "04007a0d39778621266e2208a22c4fac", "2d296c202d36b4a53f1eaddda274e4a1", - "c87806c220d125c7563c2928e836fbbd", "339b49710a0099087e51ab5afc8d8713", - "c90fbc020afd9327bf35dccae099bf77", "BAB0E5918AB2F7354D9BCA4E9A927C0F", - ]; - - private static string[] Digests8x16 = [ - "3c5a4574d96b5bb1013429636554e761", "8cf56b17c52d25eb785685f2ab48b194", - "7911e2e02abfbe226f17529ac5db08fc", "064e509948982f66a14293f406d88d42", - "5c443aa713891406d5be3af4b3cf67c6", "5d2cb98e532822ca701110cda9ada968", - "3d58836e17918b8890012dd96b95bb9d", "20e8d61ddc451b9e553a294073349ffd", - "a9aa6cf9d0dcf1977a1853ccc264e40b", "D84707DAAF6CA99F041A670F16885CBA", - ]; - - private static string[] Digests8x32 = [ - "b393a2db7a76acaccc39e04d9dc3e8ac", "bbda713ee075a7ef095f0f479b5a1f82", - "f337dce3980f70730d6f6c2c756e3b62", "796189b05dc026e865c9e95491b255d1", - "ea932c21e7189eeb215c1990491320ab", "a9fffdf9455eba5e3b01317cae140289", - "9525dbfdbf5fba61ef9c7aa5fe887503", "8c6a7e3717ff8a459f415c79bb17341c", - "3761071bfaa2363a315fe07223f95a2d", "25DADBA40EBC6C4616A7A536AC48CAA6", - ]; - - private static string[] Digests16x4 = [ - "1c0a950b3ac500def73b165b6a38467c", "95e7f7300f19da280c6a506e40304462", - "28a6af15e31f76d3ff189012475d78f5", "e330d67b859bceef62b96fc9e1f49a34", - "36eca3b8083ce2fb5f7e6227dfc34e71", "08f567d2abaa8e83e4d9b33b3f709538", - "dc2d0ba13aa9369446932f03b53dc77d", "9ab342944c4b1357aa79d39d7bebdd3a", - "77ec278c5086c88b91d68eef561ed517", "E4726C9838383FAC75A028303808EABF", - ]; - - private static string[] Digests16x8 = [ - "053a2bc4b5b7287fee524af4e77f077a", "619b720b13f14f32391a99ea7ff550d5", - "728d61c11b06baf7fe77881003a918b9", "889997b89a44c9976cb34f573e2b1eea", - "b43bfc31d1c770bb9ca5ca158c9beec4", "9d3fe9f762e0c6e4f114042147c50c7f", - "c74fdd7c9938603b01e7ecf9fdf08d61", "870c7336db1102f80f74526bd5a7cf4e", - "3fd5354a6190903d6a0b661fe177daf6", "02BB1B12EC2CD3D9E08706D177309B25", - ]; - - private static string[] Digests16x16 = [ - "1fa9e2086f6594bda60c30384fbf1635", "2098d2a030cd7c6be613edc74dc2faf8", - "f3c72b0c8e73f1ddca04d14f52d194d8", "6b31f2ee24cf88d3844a2fc67e1f39f3", - "d91a22a83575e9359c5e4871ab30ddca", "24c32a0d38b4413d2ef9bf1f842c8634", - "6e9e47bf9da9b2b9ae293e0bbd8ff086", "968b82804b5200b074bcdba9718140d4", - "4e6d7e612c5ae0bbdcc51a453cd1db3f", "D182AFFEC5DBD0C7C0BD945AF090C48D", - ]; - - private static string[] Digests16x32 = [ - "01afd04432026ff56327d6226b720be2", "a6e7be906cc6f1e7a520151bfa7c303d", - "bc05c46f18d0638f0228f1de64f07cd5", "204e613e429935f721a5b29cec7d44bb", - "aa0a7c9a7482dfc06d9685072fc5bafd", "ffb60f090d83c624bb4f7dc3a630ac4f", - "36bcb9ca9bb5eac520b050409de25da5", "34d9a5dd3363668391bc3bd05b468182", - "1e149c28db8b234e43931c347a523794", "1C0AF3C1B39A4C1866440D4C80A8CE8E", - ]; - - private static string[] Digests16x64 = [ - "727797ef15ccd8d325476fe8f12006a3", "f77c544ac8035e01920deae40cee7b07", - "12b0c69595328c465e0b25e0c9e3e9fc", "3b2a053ee8b05a8ac35ad23b0422a151", - "f3be77c0fe67eb5d9d515e92bec21eb7", "f1ece6409e01e9dd98b800d49628247d", - "efd2ec9bfbbd4fd1f6604ea369df1894", "ec703de918422b9e03197ba0ed60a199", - "739418efb89c07f700895deaa5d0b3e3", "272C2EAFEABCDA41F27EFBB48B938211", - ]; - - private static string[] Digests32x8 = [ - "4da55401331ed98acec0c516d7307513", "0ae6f3974701a5e6c20baccd26b4ca52", - "79b799f1eb77d5189535dc4e18873a0e", "90e943adf3de4f913864dce4e52b4894", - "5e1b9cc800a89ef45f5bdcc9e99e4e96", "3103405df20d254cbf32ac30872ead4b", - "648550e369b77687bff3c7d6f249b02f", "f9f73bcd8aadfc059fa260325df957a1", - "204cef70d741c25d4fe2b1d10d2649a5", "A71D39B79759B44AF95E3C03D2D72C56", - ]; - - private static string[] Digests32x16 = [ - "86ad1e1047abaf9959150222e8f19593", "1908cbe04eb4e5c9d35f1af7ffd7ee72", - "6ad3bb37ebe8374b0a4c2d18fe3ebb6a", "08d3cfe7a1148bff55eb6166da3378c6", - "656a722394764d17b6c42401b9e0ad3b", "4aa00c192102efeb325883737e562f0d", - "9881a90ca88bca4297073e60b3bb771a", "8cd74aada398a3d770fc3ace38ecd311", - "0a927e3f5ff8e8338984172cc0653b13", "B48D4391B5637D085F128C7137739400", - ]; - - private static string[] Digests32x32 = [ - "1303ca680644e3d8c9ffd4185bb2835b", "2a4d9f5cc8da307d4cf7dc021df10ba9", - "ced60d3f4e4b011a6a0314dd8a4b1fd8", "ced60d3f4e4b011a6a0314dd8a4b1fd8", - "1464b01aa928e9bd82c66bad0f921693", "90deadfb13d7c3b855ba21b326c1e202", - "af96a74f8033dff010e53a8521bc6f63", "9f1039f2ef082aaee69fcb7d749037c2", - "3f82893e478e204f2d254b34222d14dc", "9B2A16A6331AA6EC41636B8F709B39C7", - ]; - - private static string[] Digests32x64 = [ - "e1e8ed803236367821981500a3d9eebe", "0f46d124ba9f48cdd5d5290acf786d6d", - "4e2a2cfd8f56f15939bdfc753145b303", "0ce332b343934b34cd4417725faa85cb", - "1d2f8e48e3adb7c448be05d9f66f4954", "9fb2e176636a5689b26f73ca73fcc512", - "e720ebccae7e25e36f23da53ae5b5d6a", "86fe4364734169aaa4520d799890d530", - "b1870290764bb1b100d1974e2bd70f1d", "2579B3CD9B1252A282C79C36B47FA186", - ]; - - private static string[] Digests64x16 = [ - "de1b736e9d99129609d6ef3a491507a0", "516d8f6eb054d74d150e7b444185b6b9", - "69e462c3338a9aaf993c3f7cfbc15649", "821b76b1494d4f84d20817840f719a1a", - "fd9b4276e7affe1e0e4ce4f428058994", "cd82fd361a4767ac29a9f406b480b8f3", - "2792c2f810157a4a6cb13c28529ff779", "1220442d90c4255ba0969d28b91e93a6", - "c7253e10b45f7f67dfee3256c9b94825", "103DA3694F8D5C6F79CA854418F3E3F9", - ]; - - private static string[] Digests64x32 = [ - "e48e1ac15e97191a8fda08d62fff343e", "80c15b303235f9bc2259027bb92dfdc4", - "538424b24bd0830f21788e7238ca762f", "a6c5aeb722615089efbca80b02951ceb", - "12604b37875533665078405ef4582e35", "0048afa17bd3e1632d68b96048836530", - "07a0cfcb56a5eed50c4bd6c26814336b", "529d8a070de5bc6531fa3ee8f450c233", - "33c50a11c7d78f72434064f634305e95", "98BEB315BF0A9B734D4FB137FDE00DF4", - ]; - - private static string[] Digests64x64 = [ - "a1650dbcd56e10288c3e269eca37967d", "be91585259bc37bf4dc1651936e90b3e", - "afe020786b83b793c2bbd9468097ff6e", "6e1094fa7b50bc813aa2ba29f5df8755", - "9e5c34f3797e0cdd3cd9d4c05b0d8950", "bc87be7ac899cc6a28f399d7516c49fe", - "9811fd0d2dd515f06122f5d1bd18b784", "3c140e466f2c2c0d9cb7d2157ab8dc27", - "9543de76c925a8f6adc884cc7f98dc91", "73BBEC1CDFFF6D66318FB43628EA42C2", - ]; - - [Theory] - [MemberData(nameof(GetTransformSizes))] - public void VerifyDcFill(int _, int width, int height) - { - // Assign - byte[] destination = new byte[width * height]; - byte[] left = new byte[1]; - byte[] above = new byte[1]; - byte expected = 0x80; - - // Act - Av1DcFillPredictor predictor = new(new Size(width, height)); - predictor.PredictScalar(destination, (nuint)width, above, left); - - // Assert - Assert.All(destination, (b) => AssertValue(expected, b)); - } + /// + /// The offset within directional reference storage that leaves readable samples before both edge origins. + /// + private const int ReferenceOrigin = 128; + + /// + /// The hardware configurations required to exercise each SIMD tier and the complete scalar fallback. + /// + private const HwIntrinsics PredictorConfigurations = + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + + /// + /// Cardinal, base, and adjusted angles covering every directional projection zone. + /// + private static ReadOnlySpan DirectionalAngles => [36, 45, 54, 67, 90, 104, 113, 126, 135, 148, 157, 166, 180, 194, 203, 212]; + + /// + /// The complete set of AV1 filter-intra coefficient modes. + /// + private static ReadOnlySpan FilterIntraModes => + [ + Av1FilterIntraMode.DC, + Av1FilterIntraMode.Vertical, + Av1FilterIntraMode.Horizontal, + Av1FilterIntraMode.Directional157, + Av1FilterIntraMode.Paeth, + ]; + + /// + /// Verifies DC prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void DcPredictorsMatchScalarDefinitionsAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateDcPredictors, PredictorConfigurations); - [Theory] - [MemberData(nameof(GetTransformSizes))] - public void VerifyDc(int _, int width, int height) - { - // Assign - byte[] destination = new byte[width * height]; - byte[] left = new byte[height]; - byte[] above = new byte[width]; - Array.Fill(left, (byte)5); - Array.Fill(above, (byte)28); - int count = width + height; - int sum = Sum(left, height) + Sum(above, width); - byte expected = (byte)((sum + (count >> 1)) / count); - - // Act - Av1DcPredictor predictor = new(new Size(width, height)); - predictor.PredictScalar(destination, (nuint)width, above, left); - - // Assert - Assert.Equal((5 * height) + (28 * width), sum); - Assert.All(destination, (b) => AssertValue(expected, b)); - } + /// + /// Verifies horizontal prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void HorizontalPredictorMatchesScalarDefinitionAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateHorizontalPredictor, PredictorConfigurations); - [Theory] - [MemberData(nameof(GetTransformSizes))] - public void VerifyDcLeft(int _, int width, int height) - { - // Assign - byte[] destination = new byte[width * height]; - byte[] left = new byte[height]; - byte[] above = new byte[width]; - Array.Fill(left, (byte)5); - Array.Fill(above, (byte)28); - byte expected = left[0]; - - // Act - Av1DcLeftPredictor predictor = new(new Size(width, height)); - predictor.PredictScalar(destination, (nuint)width, above, left); - - // Assert - Assert.All(destination, (b) => AssertValue(expected, b)); - } + /// + /// Verifies vertical prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void VerticalPredictorMatchesScalarDefinitionAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateVerticalPredictor, PredictorConfigurations); + + /// + /// Verifies Paeth prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void PaethPredictorMatchesScalarDefinitionAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidatePaethPredictor, PredictorConfigurations); + + /// + /// Verifies smooth prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void SmoothPredictorMatchesScalarDefinitionAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateSmoothPredictor, PredictorConfigurations); + + /// + /// Verifies horizontal smooth prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void SmoothHorizontalPredictorMatchesScalarDefinitionAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateSmoothHorizontalPredictor, PredictorConfigurations); + + /// + /// Verifies vertical smooth prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void SmoothVerticalPredictorMatchesScalarDefinitionAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateSmoothVerticalPredictor, PredictorConfigurations); + + /// + /// Verifies directional prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void DirectionalPredictorsMatchScalarDefinitionsAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateDirectionalPredictors, PredictorConfigurations); + + /// + /// Verifies filter-intra prediction with each register-width tier and the scalar fallback. + /// + [Fact] + public void FilterIntraPredictorsMatchScalarDefinitionsAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateFilterIntraPredictors, PredictorConfigurations); + + /// + /// Verifies intra-edge upsampling with Vector128 and the scalar fallback. + /// + [Fact] + public void EdgeUpsamplingMatchesScalarDefinitionsAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateEdgeUpsampling, HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic); + + /// + /// Verifies intra-edge filtering with Vector128 and the scalar fallback. + /// + [Fact] + public void EdgeFilteringMatchesScalarDefinitionsAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateEdgeFiltering, HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic); - [Theory] - [MemberData(nameof(GetTransformSizes))] - public void VerifyDcTop(int _, int width, int height) + /// + /// Verifies all four DC neighbor-availability combinations at every AV1 transform size. + /// + private static void ValidateDcPredictors() { - // Assign - byte[] destination = new byte[width * height]; - byte[] left = new byte[height]; - byte[] above = new byte[width]; - Array.Fill(left, (byte)5); - Array.Fill(above, (byte)28); - byte expected = above[0]; - - // Act - Av1DcTopPredictor predictor = new(new Size(width, height)); - predictor.PredictScalar(destination, (nuint)width, above, left); - - // Assert - Assert.All(destination, (b) => AssertValue(expected, b)); + for (int sizeIndex = 0; sizeIndex < (int)Av1TransformSize.AllSizes; sizeIndex++) + { + Av1TransformSize transformSize = (Av1TransformSize)sizeIndex; + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int stride = width + 5; + byte[] above = CreateByteSamples(width, 17); + byte[] left = CreateByteSamples(height, 43); + short[] aboveHigh = CreateHighBitDepthSamples(width, 17); + short[] leftHigh = CreateHighBitDepthSamples(height, 43); + + for (int availability = 0; availability < 4; availability++) + { + bool hasLeft = (availability & 1) != 0; + bool hasAbove = (availability & 2) != 0; + byte[] expected = CreateByteDestination(stride, height); + byte[] actual = CreateByteDestination(stride, height); + short[] expectedHigh = CreateHighBitDepthDestination(stride, height); + short[] actualHigh = CreateHighBitDepthDestination(stride, height); + + Av1DcIntraPredictor.PredictScalar(hasLeft, hasAbove, expected, stride, above, left, width, height); + Av1DcIntraPredictor.Predict(hasLeft, hasAbove, actual, stride, above, left, width, height); + Av1DcIntraPredictor.PredictScalar(hasLeft, hasAbove, expectedHigh, stride, aboveHigh, leftHigh, width, height, 12); + Av1DcIntraPredictor.Predict(hasLeft, hasAbove, actualHigh, stride, aboveHigh, leftHigh, width, height, 12); + + Assert.Equal(expected, actual); + Assert.Equal(expectedHigh, actualHigh); + } + } } - [Theory] - [MemberData(nameof(GetTransformSizes))] - public void VerifySmooth(int index, int width, int height) + /// + /// Verifies horizontal prediction at every AV1 transform size and sample precision. + /// + private static void ValidateHorizontalPredictor() => ValidateNonDirectionalPredictor(Av1PredictionMode.Horizontal); + + /// + /// Verifies vertical prediction at every AV1 transform size and sample precision. + /// + private static void ValidateVerticalPredictor() => ValidateNonDirectionalPredictor(Av1PredictionMode.Vertical); + + /// + /// Verifies Paeth prediction at every AV1 transform size and sample precision. + /// + private static void ValidatePaethPredictor() => ValidateNonDirectionalPredictor(Av1PredictionMode.Paeth); + + /// + /// Verifies smooth prediction at every AV1 transform size and sample precision. + /// + private static void ValidateSmoothPredictor() => ValidateNonDirectionalPredictor(Av1PredictionMode.Smooth); + + /// + /// Verifies horizontal smooth prediction at every AV1 transform size and sample precision. + /// + private static void ValidateSmoothHorizontalPredictor() => ValidateNonDirectionalPredictor(Av1PredictionMode.SmoothHorizontal); + + /// + /// Verifies vertical smooth prediction at every AV1 transform size and sample precision. + /// + private static void ValidateSmoothVerticalPredictor() => ValidateNonDirectionalPredictor(Av1PredictionMode.SmoothVertical); + + /// + /// Verifies one closed non-directional operator at every AV1 transform size and sample precision. + /// + /// The prediction mode to verify. + private static void ValidateNonDirectionalPredictor(Av1PredictionMode mode) { - // Arrange - string expectedDigest = GetExpectedDigext((Av1TransformSize)index, Av1PredictionMode.Smooth); - Av1IntraPredictionMemory predictorMemory = new(8); - predictorMemory.Scramble(new Random(42)); - predictorMemory.CopySourceToDestination(); - const int stride = 1 << (Av1Constants.MaxSuperBlockSizeLog2 - 1); - Av1SmoothPredictor predictor = new(new Size(width, height)); - - // Act - predictor.PredictScalar(predictorMemory.Destination, stride, predictorMemory.Top, predictorMemory.Left); - - // Assert - Assert.Equal(expectedDigest, predictorMemory.GetDestinationDigest()); + Av1IntraPredictorBase predictor = Av1IntraPredictorBase.GetPredictor(mode); + for (int sizeIndex = 0; sizeIndex < (int)Av1TransformSize.AllSizes; sizeIndex++) + { + Av1TransformSize transformSize = (Av1TransformSize)sizeIndex; + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int stride = width + 5; + byte[] aboveStorage = CreateByteSamples(width + 1, 19); + byte[] left = CreateByteSamples(height, 71); + short[] aboveHighStorage = CreateHighBitDepthSamples(width + 1, 19); + short[] leftHigh = CreateHighBitDepthSamples(height, 71); + byte[] expected = CreateByteDestination(stride, height); + byte[] actual = CreateByteDestination(stride, height); + short[] expectedHigh = CreateHighBitDepthDestination(stride, height); + short[] actualHigh = CreateHighBitDepthDestination(stride, height); + + predictor.PredictScalar(expected, stride, aboveStorage.AsSpan(1), left, width, height); + predictor.Predict(actual, stride, aboveStorage.AsSpan(1), left, width, height); + predictor.PredictScalar(expectedHigh, stride, aboveHighStorage.AsSpan(1), leftHigh, width, height); + predictor.Predict(actualHigh, stride, aboveHighStorage.AsSpan(1), leftHigh, width, height); + + Assert.Equal(expected, actual); + Assert.Equal(expectedHigh, actualHigh); + } + + ValidateKnownNonDirectionalVector(mode, predictor); } - [Fact] - public void PaethFactoryUsesNearestNeighborPrediction() + /// + /// Verifies one non-directional operator against a byte-exact reference block and its translated high-bit-depth equivalent. + /// + /// The prediction mode being verified. + /// The closed operator-driven predictor. + private static void ValidateKnownNonDirectionalVector(Av1PredictionMode mode, Av1IntraPredictorBase predictor) { - byte[] destination = new byte[16]; - byte[] aboveData = [50, 60, 10, 90, 40]; - Span above = aboveData.AsSpan(1); - byte[] left = [20, 80, 30, 100]; - byte[] expected = - [ - 20, 10, 50, 20, - 80, 50, 90, 80, - 30, 10, 90, 30, - 100, 50, 100, 100, - ]; + byte[] aboveStorage; + byte[] left; + byte[] expected; - Av1PredictorFactory.GeneralPredictor( - Av1PredictionMode.Paeth, - Av1TransformSize.Size4x4, - destination, - 4, - above, - left); + if (mode == Av1PredictionMode.Paeth) + { + aboveStorage = [50, 60, 10, 90, 40]; + left = [20, 80, 30, 100]; + expected = + [ + 20, 10, 50, 20, + 80, 50, 90, 80, + 30, 10, 90, 30, + 100, 50, 100, 100, + ]; + } + else + { + aboveStorage = [0, 20, 40, 60, 80]; + left = [20, 40, 60, 80]; + expected = mode switch + { + Av1PredictionMode.Horizontal => + [ + 20, 20, 20, 20, + 40, 40, 40, 40, + 60, 60, 60, 60, + 80, 80, 80, 80, + ], + Av1PredictionMode.Vertical => + [ + 20, 40, 60, 80, + 20, 40, 60, 80, + 20, 40, 60, 80, + 20, 40, 60, 80, + ], + Av1PredictionMode.Smooth => + [ + 20, 43, 60, 73, + 43, 57, 68, 75, + 60, 68, 73, 78, + 73, 75, 78, 80, + ], + Av1PredictionMode.SmoothHorizontal => + [ + 20, 45, 60, 65, + 40, 57, 67, 70, + 60, 68, 73, 75, + 80, 80, 80, 80, + ], + _ => + [ + 20, 40, 60, 80, + 45, 57, 68, 80, + 60, 67, 73, 80, + 65, 70, 75, 80, + ], + }; + } - Assert.Equal(expected, destination); - } + byte[] actual = new byte[16]; + predictor.Predict(actual, 4, aboveStorage.AsSpan(1), left, 4, 4); - [Fact] - public void SmoothHorizontalUsesSingleAxisNormalization() - { - byte[] destination = new byte[16]; - byte[] above = [20, 40, 60, 80]; - byte[] left = [20, 40, 60, 80]; - byte[] expected = - [ - 20, 45, 60, 65, - 40, 57, 67, 70, - 60, 68, 73, 75, - 80, 80, 80, 80, - ]; + Assert.Equal(expected, actual); - Av1SmoothHorizontalPredictor.PredictScalar( - Av1TransformSize.Size4x4, - destination, - 4, - above, - left); + const int offset = 512; + short[] aboveHighStorage = new short[aboveStorage.Length]; + short[] leftHigh = new short[left.Length]; + short[] expectedHigh = new short[expected.Length]; + short[] actualHigh = new short[16]; + + for (int i = 0; i < aboveStorage.Length; i++) + { + aboveHighStorage[i] = (short)(aboveStorage[i] + offset); + } - Assert.Equal(expected, destination); + for (int i = 0; i < left.Length; i++) + { + leftHigh[i] = (short)(left[i] + offset); + } + + for (int i = 0; i < expected.Length; i++) + { + expectedHigh[i] = (short)(expected[i] + offset); + } + + predictor.Predict(actualHigh, 4, aboveHighStorage.AsSpan(1), leftHigh, 4, 4); + + Assert.Equal(expectedHigh, actualHigh); } - [Fact] - public void SmoothVerticalUsesSingleAxisNormalization() + /// + /// Verifies every directional zone, rectangular transpose, and edge-upsampling index rule. + /// + private static void ValidateDirectionalPredictors() { - byte[] destination = new byte[16]; - byte[] above = [20, 40, 60, 80]; - byte[] left = [20, 40, 60, 80]; - byte[] expected = - [ - 20, 40, 60, 80, - 45, 57, 68, 80, - 60, 67, 73, 80, - 65, 70, 75, 80, - ]; + byte[] aboveStorage = CreateByteSamples(512, 23); + byte[] leftStorage = CreateByteSamples(512, 89); + short[] aboveHighStorage = CreateHighBitDepthSamples(512, 23); + short[] leftHighStorage = CreateHighBitDepthSamples(512, 89); + ReadOnlySpan above = aboveStorage.AsSpan(ReferenceOrigin); + ReadOnlySpan left = leftStorage.AsSpan(ReferenceOrigin); + ReadOnlySpan aboveHigh = aboveHighStorage.AsSpan(ReferenceOrigin); + ReadOnlySpan leftHigh = leftHighStorage.AsSpan(ReferenceOrigin); + + foreach (int angle in DirectionalAngles) + { + for (int sizeIndex = 0; sizeIndex < (int)Av1TransformSize.AllSizes; sizeIndex++) + { + ValidateDirectionalCase((Av1TransformSize)sizeIndex, angle, false, false, above, left, aboveHigh, leftHigh); + } + } - Av1SmoothVerticalPredictor.PredictScalar( - Av1TransformSize.Size4x4, - destination, - 4, - above, - left); + // Edge upsampling is permitted only for small blocks. These cases exercise top-only, both-edge, + // and left-only indexing without asking an invalid large transform to consume an upsampled edge. + ValidateDirectionalCase(Av1TransformSize.Size4x4, 45, true, false, above, left, aboveHigh, leftHigh); + ValidateDirectionalCase(Av1TransformSize.Size4x4, 135, true, true, above, left, aboveHigh, leftHigh); + ValidateDirectionalCase(Av1TransformSize.Size4x4, 203, false, true, above, left, aboveHigh, leftHigh); + ValidateKnownDirectionalVectors(); + } - Assert.Equal(expected, destination); + /// + /// Verifies all three projection zones against byte-exact reference blocks. + /// + private static void ValidateKnownDirectionalVectors() + { + ValidateKnownDirectionalVector( + 45, + [0, 10, 20, 30, 40, 50, 60, 70, 80, 90], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ + 20, 30, 40, 50, + 30, 40, 50, 60, + 40, 50, 60, 70, + 50, 60, 70, 80, + ]); + + ValidateKnownDirectionalVector( + 135, + [5, 10, 20, 30, 40, 50, 60, 70, 80, 90], + [5, 50, 60, 70, 80, 90, 100, 110, 120, 130], + [ + 5, 10, 20, 30, + 50, 5, 10, 20, + 60, 50, 5, 10, + 70, 60, 50, 5, + ]); + + ValidateKnownDirectionalVector( + 203, + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 10, 20, 30, 40, 50, 60, 70, 80, 90], + [ + 14, 18, 23, 27, + 24, 28, 33, 37, + 34, 38, 43, 47, + 44, 48, 53, 57, + ]); } - [Fact] - public void DirectionalZone2StaticPredictorForwardsLeftUpsampling() + /// + /// Verifies one directional projection and its translated high-bit-depth equivalent. + /// + /// The adjusted directional angle. + /// The top-left prefix followed by the top reference. + /// The top-left prefix followed by the left reference. + /// The byte-exact predicted block. + private static void ValidateKnownDirectionalVector( + int angle, + ReadOnlySpan aboveStorage, + ReadOnlySpan leftStorage, + ReadOnlySpan expected) { + ReadOnlySpan above = aboveStorage[1..]; + ReadOnlySpan left = leftStorage[1..]; + byte[] actual = new byte[16]; - byte[] expected = new byte[16]; - byte[] aboveData = new byte[128]; - byte[] leftData = new byte[128]; - for (int i = 0; i < aboveData.Length; i++) + byte[] scratch = new byte[Av1DirectionalIntraPredictor.ScratchLength]; + Av1DirectionalIntraPredictor.Predict(actual, 4, Av1TransformSize.Size4x4, above, left, false, false, angle, scratch); + + Assert.Equal(expected, actual); + + const int offset = 512; + short[] aboveHighStorage = new short[aboveStorage.Length]; + short[] leftHighStorage = new short[leftStorage.Length]; + short[] expectedHigh = new short[expected.Length]; + short[] actualHigh = new short[16]; + short[] scratchHigh = new short[Av1DirectionalIntraPredictor.ScratchLength]; + + for (int i = 0; i < aboveStorage.Length; i++) + { + aboveHighStorage[i] = (short)(aboveStorage[i] + offset); + } + + for (int i = 0; i < leftStorage.Length; i++) { - aboveData[i] = (byte)((i * 5) + 1); - leftData[i] = (byte)((i * 7) + 3); + leftHighStorage[i] = (short)(leftStorage[i] + offset); } - Span above = aboveData.AsSpan(64); - Span left = leftData.AsSpan(64); - Av1DirectionalZone2Predictor predictor = new(Av1TransformSize.Size4x4); - predictor.PredictScalar(expected, 4, above, left, false, true, 64, 64); + for (int i = 0; i < expected.Length; i++) + { + expectedHigh[i] = (short)(expected[i] + offset); + } - Av1DirectionalZone2Predictor.PredictScalar( - Av1TransformSize.Size4x4, - actual, + Av1DirectionalIntraPredictor.Predict( + actualHigh, 4, - above, - left, + Av1TransformSize.Size4x4, + aboveHighStorage.AsSpan(1), + leftHighStorage.AsSpan(1), false, - true, - 64, - 64); + false, + angle, + scratchHigh); - Assert.Equal(expected, actual); + Assert.Equal(expectedHigh, actualHigh); } - [Theory] - [MemberData(nameof(GetFilterIntraPredictions))] - public void FilterIntraMatchesLibaomScalarVector(int mode, byte[] expected) + /// + /// Verifies one directional prediction configuration for both native sample representations. + /// + private static void ValidateDirectionalCase( + Av1TransformSize transformSize, + int angle, + bool upsampleAbove, + bool upsampleLeft, + ReadOnlySpan above, + ReadOnlySpan left, + ReadOnlySpan aboveHigh, + ReadOnlySpan leftHigh) { - byte[] destination = new byte[16]; - byte[] aboveData = [17, 30, 70, 110, 150]; - Span above = aboveData.AsSpan(1); - byte[] left = [40, 80, 120, 160]; - - Av1PredictorFactory.FilterIntraPredictor( - destination, - 4, - Av1TransformSize.Size4x4, - above, - left, - (Av1FilterIntraMode)mode); + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int stride = width + 5; + byte[] expected = CreateByteDestination(stride, height); + byte[] actual = CreateByteDestination(stride, height); + short[] expectedHigh = CreateHighBitDepthDestination(stride, height); + short[] actualHigh = CreateHighBitDepthDestination(stride, height); + byte[] scratch = new byte[Av1DirectionalIntraPredictor.ScratchLength]; + short[] scratchHigh = new short[Av1DirectionalIntraPredictor.ScratchLength]; + + Av1DirectionalIntraPredictor.PredictScalar(expected, stride, transformSize, above, left, upsampleAbove, upsampleLeft, angle); + Av1DirectionalIntraPredictor.Predict(actual, stride, transformSize, above, left, upsampleAbove, upsampleLeft, angle, scratch); + Av1DirectionalIntraPredictor.PredictScalar(expectedHigh, stride, transformSize, aboveHigh, leftHigh, upsampleAbove, upsampleLeft, angle); + Av1DirectionalIntraPredictor.Predict(actualHigh, stride, transformSize, aboveHigh, leftHigh, upsampleAbove, upsampleLeft, angle, scratchHigh); - Assert.Equal(expected, destination); + Assert.Equal(expected, actual); + Assert.Equal(expectedHigh, actualHigh); } - [Theory] - [MemberData(nameof(GetFilterIntraTransformSizes))] - public void FilterIntraSupportsEveryPermittedTransformSize(int transformSizeIndex) + /// + /// Verifies every filter-intra operator at each transform size permitted by the AV1 syntax. + /// + private static void ValidateFilterIntraPredictors() { - Av1TransformSize transformSize = (Av1TransformSize)transformSizeIndex; - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - byte[] destination = new byte[width * height]; - byte[] aboveData = new byte[width + 1]; - byte[] left = new byte[height]; - Array.Fill(aboveData, (byte)73); - Array.Fill(left, (byte)73); - - Av1PredictorFactory.FilterIntraPredictor( - destination, - (nuint)width, - transformSize, - aboveData.AsSpan(1), - left, - Av1FilterIntraMode.DC); - - Assert.All(destination, value => Assert.Equal(73, value)); + foreach (Av1FilterIntraMode mode in FilterIntraModes) + { + Av1FilterIntraPredictorBase predictor = Av1FilterIntraPredictorBase.GetPredictor(mode); + for (int sizeIndex = 0; sizeIndex < (int)Av1TransformSize.AllSizes; sizeIndex++) + { + Av1TransformSize transformSize = (Av1TransformSize)sizeIndex; + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + if (width > 32 || height > 32) + { + continue; + } + + int stride = width + 5; + byte[] aboveStorage = CreateByteSamples(width + 1, 29); + byte[] left = CreateByteSamples(height, 97); + short[] aboveHighStorage = CreateHighBitDepthSamples(width + 1, 29); + short[] leftHigh = CreateHighBitDepthSamples(height, 97); + byte[] expected = CreateByteDestination(stride, height); + byte[] actual = CreateByteDestination(stride, height); + short[] expectedHigh = CreateHighBitDepthDestination(stride, height); + short[] actualHigh = CreateHighBitDepthDestination(stride, height); + byte[] expectedScratch = new byte[Av1FilterIntraPredictorBase.ScratchLength]; + byte[] actualScratch = new byte[Av1FilterIntraPredictorBase.ScratchLength]; + short[] expectedHighScratch = new short[Av1FilterIntraPredictorBase.ScratchLength]; + short[] actualHighScratch = new short[Av1FilterIntraPredictorBase.ScratchLength]; + + predictor.PredictScalar(expected, stride, aboveStorage.AsSpan(1), left, width, height, expectedScratch); + predictor.Predict(actual, stride, aboveStorage.AsSpan(1), left, width, height, actualScratch); + predictor.PredictScalar(expectedHigh, stride, aboveHighStorage.AsSpan(1), leftHigh, width, height, 12, expectedHighScratch); + predictor.Predict(actualHigh, stride, aboveHighStorage.AsSpan(1), leftHigh, width, height, 12, actualHighScratch); + + Assert.Equal(expected, actual); + Assert.Equal(expectedHigh, actualHigh); + } + } + + ValidateKnownFilterIntraVectors(); } - [Theory] - [InlineData(true, true, 800)] - [InlineData(true, false, 700)] - [InlineData(false, true, 900)] - [InlineData(false, false, 512)] - public void HighBitDepthDcPredictionUsesAvailableEdges(bool hasLeft, bool hasAbove, short expected) + /// + /// Retains byte-exact libaom vectors so scalar and SIMD code cannot share the same mistranslation unnoticed. + /// + private static void ValidateKnownFilterIntraVectors() { - short[] destination = new short[16]; - short[] above = [900, 900, 900, 900]; - short[] left = [700, 700, 700, 700]; + byte[][] expectedByMode = + [ + [42, 65, 89, 123, 72, 77, 91, 110, 105, 100, 104, 112, 142, 128, 123, 124], + [44, 79, 116, 153, 69, 94, 126, 158, 94, 109, 136, 163, 119, 124, 146, 168], + [47, 67, 87, 107, 83, 93, 103, 113, 122, 127, 132, 137, 161, 163, 166, 168], + [38, 55, 81, 111, 64, 62, 73, 92, 97, 83, 81, 86, 134, 113, 103, 100], + [49, 81, 114, 148, 82, 105, 132, 159, 117, 132, 153, 174, 152, 159, 177, 190], + ]; - Av1PredictorFactory.DcPredictor( - hasLeft, - hasAbove, - Av1TransformSize.Size4x4, - destination, - 4, - above, - left, - 10); + // These edge values are the input to the five reference vectors above. The leading top value is the + // shared top-left sample addressed through above[-1] by the normative recursive filter process. + byte[] aboveStorage = [17, 30, 70, 110, 150]; + byte[] left = [40, 80, 120, 160]; + short[] aboveHighStorage = [529, 542, 582, 622, 662]; + short[] leftHigh = [552, 592, 632, 672]; - Assert.All(destination, value => Assert.Equal(expected, value)); + for (int modeIndex = 0; modeIndex < FilterIntraModes.Length; modeIndex++) + { + byte[] actual = new byte[16]; + byte[] scratch = new byte[Av1FilterIntraPredictorBase.ScratchLength]; + short[] actualHigh = new short[16]; + short[] expectedHigh = new short[16]; + short[] scratchHigh = new short[Av1FilterIntraPredictorBase.ScratchLength]; + Av1FilterIntraPredictorBase predictor = Av1FilterIntraPredictorBase.GetPredictor(FilterIntraModes[modeIndex]); + + predictor.Predict(actual, 4, aboveStorage.AsSpan(1), left, 4, 4, scratch); + + for (int i = 0; i < expectedHigh.Length; i++) + { + expectedHigh[i] = (short)(expectedByMode[modeIndex][i] + 512); + } + + predictor.Predict(actualHigh, 4, aboveHighStorage.AsSpan(1), leftHigh, 4, 4, 10, scratchHigh); + + Assert.Equal(expectedByMode[modeIndex], actual); + Assert.Equal(expectedHigh, actualHigh); + } } - [Theory] - [InlineData((int)Av1PredictionMode.Horizontal)] - [InlineData((int)Av1PredictionMode.Vertical)] - [InlineData((int)Av1PredictionMode.Paeth)] - [InlineData((int)Av1PredictionMode.Smooth)] - [InlineData((int)Av1PredictionMode.SmoothHorizontal)] - [InlineData((int)Av1PredictionMode.SmoothVertical)] - public void HighBitDepthGeneralPredictionMatchesTranslatedEightBitOracle(int modeIndex) + /// + /// Verifies vector interleaving, endpoint extension, clamping, and scalar tails in edge upsampling. + /// + private static void ValidateEdgeUpsampling() { - const int offset = 512; - Av1PredictionMode mode = (Av1PredictionMode)modeIndex; - byte[] aboveData = [50, 60, 10, 90, 40]; - byte[] left = [20, 80, 30, 100]; - short[] highAboveData = aboveData.Select(value => (short)(value + offset)).ToArray(); - short[] highLeft = left.Select(value => (short)(value + offset)).ToArray(); - byte[] expected = new byte[16]; - short[] actual = new short[16]; - - Av1PredictorFactory.GeneralPredictor( - mode, - Av1TransformSize.Size4x4, - expected, - 4, - aboveData.AsSpan(1), - left); + ReadOnlySpan counts = [4, 8, 12, 16]; + foreach (int count in counts) + { + byte[] actual = CreateUpsampleByteEdge(count); + byte[] expected = (byte[])actual.Clone(); + byte[] scratch = new byte[160]; - Av1PredictorFactory.GeneralPredictor( - mode, - Av1TransformSize.Size4x4, - actual, - 4, - highAboveData.AsSpan(1), - highLeft); + UpsampleEdgeScalar(expected, count, 8); + Av1PredictionDecoder.UpsampleIntraEdge(actual.AsSpan(2), count, scratch); + + Assert.Equal(expected, actual); + + ReadOnlySpan bitDepths = [10, 12]; - Assert.Equal(expected.Select(value => (short)(value + offset)), actual); + foreach (int bitDepth in bitDepths) + { + short[] actualHigh = CreateUpsampleHighBitDepthEdge(count, bitDepth); + short[] expectedHigh = (short[])actualHigh.Clone(); + short[] scratchHigh = new short[160]; + + UpsampleEdgeScalar(expectedHigh, count, bitDepth); + Av1PredictionDecoder.UpsampleIntraEdge(actualHigh.AsSpan(2), count, bitDepth, scratchHigh); + + Assert.Equal(expectedHigh, actualHigh); + } + } } - [Theory] - [InlineData(45)] - [InlineData(67)] - [InlineData(90)] - [InlineData(113)] - [InlineData(135)] - [InlineData(157)] - [InlineData(180)] - [InlineData(203)] - public void HighBitDepthDirectionalPredictionMatchesTranslatedEightBitOracle(int angle) + /// + /// Verifies all three edge-filter kernels across vector boundaries and the maximum normative edge length. + /// + private static void ValidateEdgeFiltering() { - const int offset = 512; - byte[] aboveData = new byte[256]; - byte[] leftData = new byte[256]; - for (int i = 0; i < aboveData.Length; i++) + ReadOnlySpan counts = [4, 8, 9, 16, 31, 64, 129]; + foreach (int count in counts) { - aboveData[i] = (byte)(20 + (i % 180)); - leftData[i] = (byte)(30 + (i % 170)); - } + for (int strength = 1; strength <= 3; strength++) + { + byte[] actual = CreateByteSamples(count, 31); + byte[] expected = (byte[])actual.Clone(); + byte[] source = (byte[])actual.Clone(); + byte[] scratch = new byte[160]; - short[] highAboveData = aboveData.Select(value => (short)(value + offset)).ToArray(); - short[] highLeftData = leftData.Select(value => (short)(value + offset)).ToArray(); - byte[] expected = new byte[16]; - short[] actual = new short[16]; + FilterEdgeScalar(source, expected, strength); + Av1PredictionDecoder.FilterIntraEdge(ref actual[0], count, strength, scratch); - Av1PredictorFactory.DirectionalPredictor( - expected, - 4, - Av1TransformSize.Size4x4, - aboveData.AsSpan(128), - leftData.AsSpan(128), - false, - false, - angle); + Assert.Equal(expected, actual); - Av1PredictorFactory.DirectionalPredictor( - actual, - 4, - Av1TransformSize.Size4x4, - highAboveData.AsSpan(128), - highLeftData.AsSpan(128), - false, - false, - angle, - 10); + short[] actualHigh = CreateHighBitDepthSamples(count, 31); + short[] expectedHigh = (short[])actualHigh.Clone(); + short[] sourceHigh = (short[])actualHigh.Clone(); + short[] scratchHigh = new short[160]; + + FilterEdgeScalar(sourceHigh, expectedHigh, strength); + Av1PredictionDecoder.FilterIntraEdge(ref actualHigh[0], count, strength, scratchHigh); - Assert.Equal(expected.Select(value => (short)(value + offset)), actual); + Assert.Equal(expectedHigh, actualHigh); + } + } } - [Theory] - [MemberData(nameof(GetFilterIntraPredictions))] - public void HighBitDepthFilterIntraMatchesTranslatedLibaomVector(int mode, byte[] expected) + /// + /// Creates deterministic 8-bit samples with enough variation to expose lane-order mistakes. + /// + private static byte[] CreateByteSamples(int length, int seed) { - const int offset = 512; - short[] destination = new short[16]; - short[] aboveData = [529, 542, 582, 622, 662]; - short[] left = [552, 592, 632, 672]; + byte[] samples = new byte[length]; + for (int i = 0; i < samples.Length; i++) + { + samples[i] = (byte)(((i * 73) + (seed * 29) + ((i * i) * 7)) & 255); + } - Av1PredictorFactory.FilterIntraPredictor( - destination, - 4, - Av1TransformSize.Size4x4, - aboveData.AsSpan(1), - left, - (Av1FilterIntraMode)mode, - 10); + return samples; + } - Assert.Equal(expected.Select(value => (short)(value + offset)), destination); + /// + /// Creates deterministic 12-bit samples with values spanning the full reconstructed range. + /// + private static short[] CreateHighBitDepthSamples(int length, int seed) + { + short[] samples = new short[length]; + for (int i = 0; i < samples.Length; i++) + { + samples[i] = (short)(((i * 977) + (seed * 131) + ((i * i) * 37)) & 4095); + } + + return samples; } - private static void AssertValue(byte expected, byte actual) + /// + /// Creates a strided byte destination initialized with a padding sentinel. + /// + private static byte[] CreateByteDestination(int stride, int height) { - Assert.NotEqual(0, actual); - Assert.Equal(expected, actual); + byte[] destination = new byte[stride * height]; + Array.Fill(destination, (byte)0xCD); + return destination; } - private static int Sum(Span values, int length) + /// + /// Creates a strided high-bit-depth destination initialized with a padding sentinel. + /// + private static short[] CreateHighBitDepthDestination(int stride, int height) { - int sum = 0; - for (int i = 0; i < length; i++) + short[] destination = new short[stride * height]; + Array.Fill(destination, (short)-1234); + return destination; + } + + /// + /// Creates an 8-bit edge with two prefix samples and room for all interleaved outputs. + /// + private static byte[] CreateUpsampleByteEdge(int count) + { + byte[] edge = new byte[(2 * count) + 4]; + Array.Fill(edge, (byte)0xA5); + edge[1] = 231; + for (int i = 0; i < count; i++) { - sum += values[i]; + edge[i + 2] = (byte)(((i * 97) + 41) & 255); } - return sum; + return edge; } - public static TheoryData GetTransformSizes() + /// + /// Creates a high-bit-depth edge containing extrema that exercise interpolation clamping. + /// + private static short[] CreateUpsampleHighBitDepthEdge(int count, int bitDepth) { - TheoryData combinations = []; - for (int s = 0; s < (int)Av1TransformSize.AllSizes; s++) + int maximum = (1 << bitDepth) - 1; + short[] edge = new short[(2 * count) + 4]; + Array.Fill(edge, (short)-1); + edge[1] = (short)maximum; + for (int i = 0; i < count; i++) { - Av1TransformSize size = (Av1TransformSize)s; - int width = size.GetWidth(); - int height = size.GetHeight(); - combinations.Add(s, width, height); + edge[i + 2] = (short)((i & 1) == 0 ? 0 : maximum); } - return combinations; + return edge; } - public static TheoryData GetFilterIntraPredictions() => new() + /// + /// Applies the normative four-tap upsampling formula to an edge stored at index two. + /// + private static void UpsampleEdgeScalar(T[] edge, int count, int bitDepth) + where T : unmanaged, IBinaryInteger { + T[] input = new T[count + 3]; + input[0] = edge[1]; + input[1] = edge[1]; + for (int i = 0; i < count; i++) { - (int)Av1FilterIntraMode.DC, - [42, 65, 89, 123, 72, 77, 91, 110, 105, 100, 104, 112, 142, 128, 123, 124] - }, - { - (int)Av1FilterIntraMode.Vertical, - [44, 79, 116, 153, 69, 94, 126, 158, 94, 109, 136, 163, 119, 124, 146, 168] - }, - { - (int)Av1FilterIntraMode.Horizontal, - [47, 67, 87, 107, 83, 93, 103, 113, 122, 127, 132, 137, 161, 163, 166, 168] - }, - { - (int)Av1FilterIntraMode.Directional157, - [38, 55, 81, 111, 64, 62, 73, 92, 97, 83, 81, 86, 134, 113, 103, 100] - }, + input[i + 2] = edge[i + 2]; + } + + input[count + 2] = input[count + 1]; + edge[0] = input[0]; + int maximum = (1 << bitDepth) - 1; + for (int i = 0; i < count; i++) { - (int)Av1FilterIntraMode.Paeth, - [49, 81, 114, 148, 82, 105, 132, 159, 117, 132, 153, 174, 152, 159, 177, 190] - }, - }; + int value = -int.CreateChecked(input[i]) + + (9 * int.CreateChecked(input[i + 1])) + + (9 * int.CreateChecked(input[i + 2])) + - int.CreateChecked(input[i + 3]); - public static TheoryData GetFilterIntraTransformSizes() + edge[(2 * i) + 1] = T.CreateChecked(Math.Clamp((value + 8) >> 4, 0, maximum)); + edge[(2 * i) + 2] = input[i + 2]; + } + } + + /// + /// Applies the normative AV1 edge-filter definition to an independent source copy. + /// + private static void FilterEdgeScalar(T[] source, T[] destination, int strength) + where T : unmanaged, IBinaryInteger { - TheoryData transformSizes = []; - for (int i = 0; i < (int)Av1TransformSize.AllSizes; i++) + ReadOnlySpan kernel = strength switch + { + 1 => [0, 4, 8, 4, 0], + 2 => [0, 5, 6, 5, 0], + _ => [2, 4, 4, 4, 2], + }; + + for (int i = 1; i < source.Length; i++) { - Av1TransformSize transformSize = (Av1TransformSize)i; - if (transformSize.GetWidth() <= 32 && transformSize.GetHeight() <= 32) + int sum = 0; + for (int tap = 0; tap < kernel.Length; tap++) { - transformSizes.Add(i); + int sourceIndex = Math.Clamp(i - 2 + tap, 0, source.Length - 1); + sum += int.CreateChecked(source[sourceIndex]) * kernel[tap]; } - } - return transformSizes; + destination[i] = T.CreateChecked((sum + 8) >> 4); + } } - - private static string GetExpectedDigext(Av1TransformSize size, Av1PredictionMode mode) => size switch - { - Av1TransformSize.Size4x4 => Digests4x4[(int)mode], - Av1TransformSize.Size8x4 => Digests8x4[(int)mode], - Av1TransformSize.Size4x8 => Digests4x8[(int)mode], - Av1TransformSize.Size8x8 => Digests8x8[(int)mode], - Av1TransformSize.Size4x16 => Digests4x16[(int)mode], - Av1TransformSize.Size16x4 => Digests16x4[(int)mode], - Av1TransformSize.Size8x16 => Digests8x16[(int)mode], - Av1TransformSize.Size16x8 => Digests16x8[(int)mode], - Av1TransformSize.Size16x16 => Digests16x16[(int)mode], - Av1TransformSize.Size8x32 => Digests8x32[(int)mode], - Av1TransformSize.Size32x8 => Digests32x8[(int)mode], - Av1TransformSize.Size16x32 => Digests16x32[(int)mode], - Av1TransformSize.Size32x16 => Digests32x16[(int)mode], - Av1TransformSize.Size32x32 => Digests32x32[(int)mode], - Av1TransformSize.Size16x64 => Digests16x64[(int)mode], - Av1TransformSize.Size64x16 => Digests64x16[(int)mode], - Av1TransformSize.Size32x64 => Digests32x64[(int)mode], - Av1TransformSize.Size64x32 => Digests64x32[(int)mode], - Av1TransformSize.Size64x64 => Digests64x64[(int)mode], - _ => string.Empty, - }; }