diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 7f898dc5a..2c5450e0e 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -65,6 +65,8 @@ Immediate checkpoint: **complete layered AV1 image-item decoding through the exi - [x] Store the eight fixed reference-validity, order-hint, and map-index tables inline on the frame header, retaining complete multi-bit order hints without per-header array allocations. - [ ] Preserve reconstruction, reference-frame, primary-CDF, segmentation, loop-filter, and motion state across every dependent layer in one image-item decoder session. - [ ] Implement the complete inter-frame entropy, mode, motion-vector, compound-prediction, inter-prediction, and warped/global-motion paths permitted by the image profile. + - [x] Implement allocation-free SIMD-first translational single-reference interpolation for regular, smooth, sharp, and bilinear filters across 8/10/12-bit samples. The predictor mirrors JPEG's closed static operator architecture, descends through `Vector512`, `Vector256`, and `Vector128` before scalar fallback, and passes the exact independent convolution oracle through `FeatureTestRunner`. + - [ ] Connect reference selection, motion-vector derivation, compound modes, warped/global motion, and reconstructed-plane writes through tile decoding, then verify them with independently encoded inter-frame AV1 image-layer fixtures. - [ ] Return the explicitly selected spatial layer or the final displayed layer, keeping reference reconstruction separate from display-only film grain. - [ ] Verify color and auxiliary-alpha output exactly against both pinned libavif progressive fixtures under normal SIMD dispatch and all required `FeatureTestRunner` fallbacks. - [ ] Correct the audited 12-bit inverse ADST4, Identity4, and Identity16 SIMD arithmetic by widening only the libaom-widened multiply/accumulate operations, with exact conformant-range vectors and `FeatureTestRunner` coverage. @@ -521,6 +523,10 @@ Implement and verify in dependency order: - [x] Intra-block copy for still-image intra frames. - [x] Decode tile-adaptive integer displacement vectors, derive and validate spatial references, apply the inter transform sets, and reconstruct luma and subsampled chroma through allocation-free `Vector512`/`Vector256`/`Vector128` operators with exact-width stores and scalar fallback. `FeatureTestRunner` verifies every transform width, bit-depth storage path, chroma phase, intrinsic tier, scalar continuation, and destination-padding boundary. - [x] Verify displacement-vector entropy, spatial candidate ordering, wavefront legality, native 8/10/12-bit reconstruction, and presented output against independently encoded opaque intra-block-copy AVIF fixtures from the pinned generic libaom/libavif reference. The fixtures require actual intra-block-copy block selection, compare every native plane sample from retained scalar-decoder Y4M output, and compare every presented RGBA byte exactly under normal hardware dispatch and each narrower fallback configuration without a tolerance. +- [ ] Inter-frame prediction for layered still-image items. + - [x] Implement allocation-free SIMD-first translational single-reference interpolation for regular, smooth, sharp, and bilinear filters; reduced four-sample kernels; horizontal, vertical, and separable two-dimensional convolution; exact AV1 rounding; 8/10/12-bit clipping; padded reference origins; and guarded destination strides. The operator contract and concrete operator files mirror JPEG color conversion, and `FeatureTestRunner` verifies normal, AVX-512-disabled, AVX-disabled, and scalar execution against an independent fixed-point oracle. + - [ ] Decode and connect reference indices, motion vectors, compound prediction, inter-intra prediction, masked blending, warped motion, global motion, and OBMC through reconstructed reference planes. + - [ ] Verify every connected inter mode and filter with independently encoded dependent-layer AV1 image-item fixtures and exact native-plane comparisons. - [ ] Lossless and high-bit-depth reconstruction with correct clipping and intermediate precision. - [x] Route lossless 4x4 blocks through allocation-free reversible inverse Walsh-Hadamard reconstruction for 8/10/12-bit samples, including the DC-only specialization, `Vector128` production traversal, scalar fallback, exact clipping, and `FeatureTestRunner` parity. - [x] Verify lossless syntax, inverse quantization, prediction, and presented reconstruction with independently encoded 8/10/12-bit AVIF fixtures. The tests require coded residuals with palette and intra-block copy disabled, compare every native YUV sample with the pinned generic libaom-backed decoder, and compare every presented RGBA byte with pinned generic libavif exactly under normal hardware dispatch and the scalar fallback. diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Arithmetic.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Arithmetic.cs new file mode 100644 index 000000000..81d83db1e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Arithmetic.cs @@ -0,0 +1,347 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Provides lane-wise convolution, rounding, clipping, and packing shared by every interpolation filter. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Convolves sixteen adjacent 8-bit samples into four signed 32-bit accumulator vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Convolve( + ref byte source, + int tapStride, + nuint column, + ref short coefficients, + int tapCount, + Vector128 initial, + out Vector128 result0, + out Vector128 result1, + out Vector128 result2, + out Vector128 result3) + { + result0 = initial; + result1 = initial; + result2 = initial; + result3 = initial; + + for (int tap = 0; tap < tapCount; tap++) + { + Vector128 samples = Vector128.LoadUnsafe(ref Unsafe.Add(ref source, tap * tapStride), column); + Av1IntraPredictorBase.Widen(samples, out Vector128 samples0, out Vector128 samples1, out Vector128 samples2, out Vector128 samples3); + Vector128 coefficient = Vector128.Create((int)Unsafe.Add(ref coefficients, tap)); + + // Each widened vector retains four consecutive source columns. Applying the same tap coefficient to all + // lanes evaluates sixteen independent finite-impulse-response filters without a horizontal reduction. + result0 += samples0 * coefficient; + result1 += samples1 * coefficient; + result2 += samples2 * coefficient; + result3 += samples3 * coefficient; + } + } + + /// + /// Convolves thirty-two adjacent 8-bit samples into four signed 32-bit accumulator vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Convolve( + ref byte source, + int tapStride, + nuint column, + ref short coefficients, + int tapCount, + Vector256 initial, + out Vector256 result0, + out Vector256 result1, + out Vector256 result2, + out Vector256 result3) + { + result0 = initial; + result1 = initial; + result2 = initial; + result3 = initial; + + for (int tap = 0; tap < tapCount; tap++) + { + Vector256 samples = Vector256.LoadUnsafe(ref Unsafe.Add(ref source, tap * tapStride), column); + Av1IntraPredictorBase.Widen(samples, out Vector256 samples0, out Vector256 samples1, out Vector256 samples2, out Vector256 samples3); + Vector256 coefficient = Vector256.Create((int)Unsafe.Add(ref coefficients, tap)); + + result0 += samples0 * coefficient; + result1 += samples1 * coefficient; + result2 += samples2 * coefficient; + result3 += samples3 * coefficient; + } + } + + /// + /// Convolves sixty-four adjacent 8-bit samples into four signed 32-bit accumulator vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Convolve( + ref byte source, + int tapStride, + nuint column, + ref short coefficients, + int tapCount, + Vector512 initial, + out Vector512 result0, + out Vector512 result1, + out Vector512 result2, + out Vector512 result3) + { + result0 = initial; + result1 = initial; + result2 = initial; + result3 = initial; + + for (int tap = 0; tap < tapCount; tap++) + { + Vector512 samples = Vector512.LoadUnsafe(ref Unsafe.Add(ref source, tap * tapStride), column); + Av1IntraPredictorBase.Widen(samples, out Vector512 samples0, out Vector512 samples1, out Vector512 samples2, out Vector512 samples3); + Vector512 coefficient = Vector512.Create((int)Unsafe.Add(ref coefficients, tap)); + + result0 += samples0 * coefficient; + result1 += samples1 * coefficient; + result2 += samples2 * coefficient; + result3 += samples3 * coefficient; + } + } + + /// + /// Convolves eight adjacent nonnegative 16-bit samples into two signed 32-bit accumulator vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Convolve( + ref short source, + int tapStride, + nuint column, + ref short coefficients, + int tapCount, + Vector128 initial, + out Vector128 result0, + out Vector128 result1) + { + result0 = initial; + result1 = initial; + + for (int tap = 0; tap < tapCount; tap++) + { + Vector128 samples = Vector128.LoadUnsafe(ref Unsafe.Add(ref source, tap * tapStride), column); + Av1IntraPredictorBase.Widen(samples, out Vector128 samples0, out Vector128 samples1); + Vector128 coefficient = Vector128.Create((int)Unsafe.Add(ref coefficients, tap)); + + // Reconstructed 10- and 12-bit samples and biased 2D intermediates are below short.MaxValue, so signed + // widening preserves their values while allowing negative interpolation coefficients. + result0 += samples0 * coefficient; + result1 += samples1 * coefficient; + } + } + + /// + /// Convolves sixteen adjacent nonnegative 16-bit samples into two signed 32-bit accumulator vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Convolve( + ref short source, + int tapStride, + nuint column, + ref short coefficients, + int tapCount, + Vector256 initial, + out Vector256 result0, + out Vector256 result1) + { + result0 = initial; + result1 = initial; + + for (int tap = 0; tap < tapCount; tap++) + { + Vector256 samples = Vector256.LoadUnsafe(ref Unsafe.Add(ref source, tap * tapStride), column); + Av1IntraPredictorBase.Widen(samples, out Vector256 samples0, out Vector256 samples1); + Vector256 coefficient = Vector256.Create((int)Unsafe.Add(ref coefficients, tap)); + result0 += samples0 * coefficient; + result1 += samples1 * coefficient; + } + } + + /// + /// Convolves thirty-two adjacent nonnegative 16-bit samples into two signed 32-bit accumulator vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Convolve( + ref short source, + int tapStride, + nuint column, + ref short coefficients, + int tapCount, + Vector512 initial, + out Vector512 result0, + out Vector512 result1) + { + result0 = initial; + result1 = initial; + + for (int tap = 0; tap < tapCount; tap++) + { + Vector512 samples = Vector512.LoadUnsafe(ref Unsafe.Add(ref source, tap * tapStride), column); + Av1IntraPredictorBase.Widen(samples, out Vector512 samples0, out Vector512 samples1); + Vector512 coefficient = Vector512.Create((int)Unsafe.Add(ref coefficients, tap)); + result0 += samples0 * coefficient; + result1 += samples1 * coefficient; + } + } + + /// + /// Applies AV1 power-of-two rounding to four-lane signed accumulators. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 RoundPowerOfTwo(Vector128 value, int bits) + => bits == 0 ? value : (value + Vector128.Create(1 << (bits - 1))) >> bits; + + /// + /// Applies AV1 power-of-two rounding to eight-lane signed accumulators. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 RoundPowerOfTwo(Vector256 value, int bits) + => bits == 0 ? value : (value + Vector256.Create(1 << (bits - 1))) >> bits; + + /// + /// Applies AV1 power-of-two rounding to sixteen-lane signed accumulators. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 RoundPowerOfTwo(Vector512 value, int bits) + => bits == 0 ? value : (value + Vector512.Create(1 << (bits - 1))) >> bits; + + /// + /// Clips and packs sixteen signed accumulators into 8-bit samples. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PackBytes(Vector128 result0, Vector128 result1, Vector128 result2, Vector128 result3) + { + Vector128 maximum = Vector128.Create((int)byte.MaxValue); + result0 = Vector128.Clamp(result0, Vector128.Zero, maximum); + result1 = Vector128.Clamp(result1, Vector128.Zero, maximum); + result2 = Vector128.Clamp(result2, Vector128.Zero, maximum); + result3 = Vector128.Clamp(result3, Vector128.Zero, maximum); + return Av1IntraPredictorBase.Narrow(result0, result1, result2, result3); + } + + /// + /// Clips and packs thirty-two signed accumulators into 8-bit samples. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PackBytes(Vector256 result0, Vector256 result1, Vector256 result2, Vector256 result3) + { + Vector256 maximum = Vector256.Create((int)byte.MaxValue); + result0 = Vector256.Clamp(result0, Vector256.Zero, maximum); + result1 = Vector256.Clamp(result1, Vector256.Zero, maximum); + result2 = Vector256.Clamp(result2, Vector256.Zero, maximum); + result3 = Vector256.Clamp(result3, Vector256.Zero, maximum); + return Av1IntraPredictorBase.Narrow(result0, result1, result2, result3); + } + + /// + /// Clips and packs sixty-four signed accumulators into 8-bit samples. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 PackBytes(Vector512 result0, Vector512 result1, Vector512 result2, Vector512 result3) + { + Vector512 maximum = Vector512.Create((int)byte.MaxValue); + result0 = Vector512.Clamp(result0, Vector512.Zero, maximum); + result1 = Vector512.Clamp(result1, Vector512.Zero, maximum); + result2 = Vector512.Clamp(result2, Vector512.Zero, maximum); + result3 = Vector512.Clamp(result3, Vector512.Zero, maximum); + return Av1IntraPredictorBase.Narrow(result0, result1, result2, result3); + } + + /// + /// Clips and packs eight signed accumulators into high-bit-depth samples. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PackHighBitDepth(Vector128 result0, Vector128 result1, int maximumValue) + { + Vector128 maximum = Vector128.Create(maximumValue); + result0 = Vector128.Clamp(result0, Vector128.Zero, maximum); + result1 = Vector128.Clamp(result1, Vector128.Zero, maximum); + return Av1IntraPredictorBase.Narrow(result0, result1).AsUInt16(); + } + + /// + /// Clips and packs sixteen signed accumulators into high-bit-depth samples. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PackHighBitDepth(Vector256 result0, Vector256 result1, int maximumValue) + { + Vector256 maximum = Vector256.Create(maximumValue); + result0 = Vector256.Clamp(result0, Vector256.Zero, maximum); + result1 = Vector256.Clamp(result1, Vector256.Zero, maximum); + return Av1IntraPredictorBase.Narrow(result0, result1).AsUInt16(); + } + + /// + /// Clips and packs thirty-two signed accumulators into high-bit-depth samples. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 PackHighBitDepth(Vector512 result0, Vector512 result1, int maximumValue) + { + Vector512 maximum = Vector512.Create(maximumValue); + result0 = Vector512.Clamp(result0, Vector512.Zero, maximum); + result1 = Vector512.Clamp(result1, Vector512.Zero, maximum); + return Av1IntraPredictorBase.Narrow(result0, result1).AsUInt16(); + } + + /// + /// Computes one signed Q7 convolution sum from 8-bit samples. + /// + private static int ConvolveScalar(ref byte source, int sourceStride, ref short coefficients, int tapCount) + { + int sum = 0; + for (int tap = 0; tap < tapCount; tap++) + { + sum += Unsafe.Add(ref coefficients, tap) * Unsafe.Add(ref source, tap * sourceStride); + } + + return sum; + } + + /// + /// Computes one signed Q7 convolution sum from high-bit-depth samples. + /// + private static int ConvolveScalar(ref ushort source, int sourceStride, ref short coefficients, int tapCount) + { + int sum = 0; + for (int tap = 0; tap < tapCount; tap++) + { + sum += Unsafe.Add(ref coefficients, tap) * Unsafe.Add(ref source, tap * sourceStride); + } + + return sum; + } + + /// + /// Computes one signed Q7 convolution sum from biased intermediate samples. + /// + private static int ConvolveScalar(ref short source, int sourceStride, ref short coefficients, int tapCount) + { + int sum = 0; + for (int tap = 0; tap < tapCount; tap++) + { + sum += Unsafe.Add(ref coefficients, tap) * Unsafe.Add(ref source, tap * sourceStride); + } + + return sum; + } + + /// + /// Rounds an integer after division by a power of two using AV1's unsigned-bias rule. + /// + private static int RoundPowerOfTwo(int value, int bits) => bits == 0 ? value : (value + (1 << (bits - 1))) >> bits; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.BilinearOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.BilinearOperator.cs new file mode 100644 index 000000000..5d81a3a37 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.BilinearOperator.cs @@ -0,0 +1,20 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Defines bilinear interpolation for translational inter prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Selects bilinear interpolation coefficients. + /// + internal readonly struct BilinearOperator : IAv1InterPredictorOperator + { + /// + public static ReadOnlySpan GetCoefficients(int phase, bool useReducedFilter) + => GetPhase(Bilinear, phase); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs new file mode 100644 index 000000000..1076323ad --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs @@ -0,0 +1,887 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Selects interpolation filters and the widest supported traversal for a translational prediction block. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Selects an 8-bit horizontal interpolation operator. + /// + private static void Dispatch( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span scratch, + bool scalarOnly) + { + switch (horizontalFilter) + { + case Av1InterpolationFilter.Regular: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Smooth: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Sharp: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + default: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + } + } + + /// + /// Selects a high-bit-depth horizontal interpolation operator. + /// + private static void Dispatch( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch, + bool scalarOnly) + { + switch (horizontalFilter) + { + case Av1InterpolationFilter.Regular: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Smooth: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Sharp: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + default: + DispatchVertical( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + } + } + + /// + /// Copies an 8-bit integer-position block using the widest vector that fits a complete row prefix. + /// + private static void Copy( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + bool scalarOnly) + { + if (scalarOnly) + { + CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); + return; + } + + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) + { + int vectorEnd = width - Vector512.Count; + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int column = 0; + + for (; column <= vectorEnd; column += Vector512.Count) + { + Vector512.LoadUnsafe(ref sourceRow, (nuint)column).StoreUnsafe(ref destinationRow, (nuint)column); + } + + for (; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + + return; + } + + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + int vectorEnd = width - Vector256.Count; + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int column = 0; + + for (; column <= vectorEnd; column += Vector256.Count) + { + Vector256.LoadUnsafe(ref sourceRow, (nuint)column).StoreUnsafe(ref destinationRow, (nuint)column); + } + + for (; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + if (width < Vector128.Count) + { + StorePartial(Vector128.LoadUnsafe(ref sourceRow), ref destinationRow, width); + continue; + } + + int vectorEnd = width - Vector128.Count; + int column = 0; + for (; column <= vectorEnd; column += Vector128.Count) + { + Vector128.LoadUnsafe(ref sourceRow, (nuint)column).StoreUnsafe(ref destinationRow, (nuint)column); + } + + for (; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + + return; + } + + CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); + } + + /// + /// Copies a high-bit-depth integer-position block using the widest vector that fits a complete row prefix. + /// + private static void Copy( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + bool scalarOnly) + { + if (scalarOnly) + { + CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); + return; + } + + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) + { + int vectorEnd = width - Vector512.Count; + for (int row = 0; row < height; row++) + { + ref ushort sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int column = 0; + + for (; column <= vectorEnd; column += Vector512.Count) + { + Vector512.LoadUnsafe(ref sourceRow, (nuint)column).StoreUnsafe(ref destinationRow, (nuint)column); + } + + for (; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + + return; + } + + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + int vectorEnd = width - Vector256.Count; + for (int row = 0; row < height; row++) + { + ref ushort sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int column = 0; + + for (; column <= vectorEnd; column += Vector256.Count) + { + Vector256.LoadUnsafe(ref sourceRow, (nuint)column).StoreUnsafe(ref destinationRow, (nuint)column); + } + + for (; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + for (int row = 0; row < height; row++) + { + ref ushort sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + if (width < Vector128.Count) + { + // Four high-bit-depth samples occupy exactly the lower 64 bits of the vector. + Vector128.LoadUnsafe(ref sourceRow).GetLower().StoreUnsafe(ref destinationRow); + continue; + } + + int vectorEnd = width - Vector128.Count; + int column = 0; + for (; column <= vectorEnd; column += Vector128.Count) + { + Vector128.LoadUnsafe(ref sourceRow, (nuint)column).StoreUnsafe(ref destinationRow, (nuint)column); + } + + for (; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + + return; + } + + CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); + } + + /// + /// Applies a one-dimensional 8-bit filter using one SIMD width for the complete block. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + bool scalarOnly) + { + if (!scalarOnly) + { + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + Vector512.Zero); + + return; + } + + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + Vector256.Zero); + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + Vector128.Zero); + + return; + } + } + + FilterDirectScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound); + } + + /// + /// Applies a one-dimensional high-bit-depth filter using one SIMD width for the complete block. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + int bitDepth, + bool scalarOnly) + { + if (!scalarOnly) + { + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth, + Vector512.Zero); + + return; + } + + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth, + Vector256.Zero); + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth, + Vector128.Zero); + + return; + } + } + + FilterDirectScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth); + } + + /// + /// Applies separable two-dimensional filtering to an 8-bit block using one SIMD width. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + Span scratch, + bool scalarOnly) + { + if (!scalarOnly) + { + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch, + Vector512.Zero); + + return; + } + + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch, + Vector256.Zero); + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch, + Vector128.Zero); + + return; + } + } + + Filter2DScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch); + } + + /// + /// Applies separable two-dimensional filtering to a high-bit-depth block using one SIMD width. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + bool scalarOnly) + { + if (!scalarOnly) + { + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch, + Vector512.Zero); + + return; + } + + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch, + Vector256.Zero); + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch, + Vector128.Zero); + + return; + } + } + + Filter2DScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch); + } + + /// + /// Copies an 8-bit integer-position prediction without explicit hardware intrinsics. + /// + private static void CopyScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + } + + /// + /// Copies a high-bit-depth integer-position prediction without explicit hardware intrinsics. + /// + private static void CopyScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + + for (int row = 0; row < height; row++) + { + ref ushort sourceRow = ref Unsafe.Add(ref sourceBase, row * sourceStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + Unsafe.Add(ref destinationRow, column) = Unsafe.Add(ref sourceRow, column); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Filters.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Filters.cs new file mode 100644 index 000000000..eba9293c5 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Filters.cs @@ -0,0 +1,153 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Provides the normative Q7 interpolation coefficients used by AV1 inter prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// The number of stored coefficient positions in every decoder interpolation kernel. + /// + private const int FilterCoefficientCount = 8; + + /// + /// Gets the regular eight-tap kernels for the sixteen subpixel phases. + /// + private static ReadOnlySpan RegularEightTap => + [ + 0, 0, 0, 128, 0, 0, 0, 0, + 0, 2, -6, 126, 8, -2, 0, 0, + 0, 2, -10, 122, 18, -4, 0, 0, + 0, 2, -12, 116, 28, -8, 2, 0, + 0, 2, -14, 110, 38, -10, 2, 0, + 0, 2, -14, 102, 48, -12, 2, 0, + 0, 2, -16, 94, 58, -12, 2, 0, + 0, 2, -14, 84, 66, -12, 2, 0, + 0, 2, -14, 76, 76, -14, 2, 0, + 0, 2, -12, 66, 84, -14, 2, 0, + 0, 2, -12, 58, 94, -16, 2, 0, + 0, 2, -12, 48, 102, -14, 2, 0, + 0, 2, -10, 38, 110, -14, 2, 0, + 0, 2, -8, 28, 116, -12, 2, 0, + 0, 0, -4, 18, 122, -10, 2, 0, + 0, 0, -2, 8, 126, -6, 2, 0, + ]; + + /// + /// Gets the smooth eight-tap kernels for the sixteen subpixel phases. + /// + private static ReadOnlySpan SmoothEightTap => + [ + 0, 0, 0, 128, 0, 0, 0, 0, + 0, 2, 28, 62, 34, 2, 0, 0, + 0, 0, 26, 62, 36, 4, 0, 0, + 0, 0, 22, 62, 40, 4, 0, 0, + 0, 0, 20, 60, 42, 6, 0, 0, + 0, 0, 18, 58, 44, 8, 0, 0, + 0, 0, 16, 56, 46, 10, 0, 0, + 0, -2, 16, 54, 48, 12, 0, 0, + 0, -2, 14, 52, 52, 14, -2, 0, + 0, 0, 12, 48, 54, 16, -2, 0, + 0, 0, 10, 46, 56, 16, 0, 0, + 0, 0, 8, 44, 58, 18, 0, 0, + 0, 0, 6, 42, 60, 20, 0, 0, + 0, 0, 4, 40, 62, 22, 0, 0, + 0, 0, 4, 36, 62, 26, 0, 0, + 0, 0, 2, 34, 62, 28, 2, 0, + ]; + + /// + /// Gets the sharp eight-tap kernels for the sixteen subpixel phases. + /// + private static ReadOnlySpan SharpEightTap => + [ + 0, 0, 0, 128, 0, 0, 0, 0, + -2, 2, -6, 126, 8, -2, 2, 0, + -2, 6, -12, 124, 16, -6, 4, -2, + -2, 8, -18, 120, 26, -10, 6, -2, + -4, 10, -22, 116, 38, -14, 6, -2, + -4, 10, -22, 108, 48, -18, 8, -2, + -4, 10, -24, 100, 60, -20, 8, -2, + -4, 10, -24, 90, 70, -22, 10, -2, + -4, 12, -24, 80, 80, -24, 12, -4, + -2, 10, -22, 70, 90, -24, 10, -4, + -2, 8, -20, 60, 100, -24, 10, -4, + -2, 8, -18, 48, 108, -22, 10, -4, + -2, 6, -14, 38, 116, -22, 10, -4, + -2, 6, -10, 26, 120, -18, 8, -2, + -2, 4, -6, 16, 124, -12, 6, -2, + 0, 2, -2, 8, 126, -6, 2, -2, + ]; + + /// + /// Gets the regular reduced kernels selected when a block dimension is at most four samples. + /// + private static ReadOnlySpan RegularFourTap => + [ + 0, 0, 0, 128, 0, 0, 0, 0, + 0, 0, -4, 126, 8, -2, 0, 0, + 0, 0, -8, 122, 18, -4, 0, 0, + 0, 0, -10, 116, 28, -6, 0, 0, + 0, 0, -12, 110, 38, -8, 0, 0, + 0, 0, -12, 102, 48, -10, 0, 0, + 0, 0, -14, 94, 58, -10, 0, 0, + 0, 0, -12, 84, 66, -10, 0, 0, + 0, 0, -12, 76, 76, -12, 0, 0, + 0, 0, -10, 66, 84, -12, 0, 0, + 0, 0, -10, 58, 94, -14, 0, 0, + 0, 0, -10, 48, 102, -12, 0, 0, + 0, 0, -8, 38, 110, -12, 0, 0, + 0, 0, -6, 28, 116, -10, 0, 0, + 0, 0, -4, 18, 122, -8, 0, 0, + 0, 0, -2, 8, 126, -4, 0, 0, + ]; + + /// + /// Gets the smooth reduced kernels selected when a block dimension is at most four samples. + /// + private static ReadOnlySpan SmoothFourTap => + [ + 0, 0, 0, 128, 0, 0, 0, 0, + 0, 0, 30, 62, 34, 2, 0, 0, + 0, 0, 26, 62, 36, 4, 0, 0, + 0, 0, 22, 62, 40, 4, 0, 0, + 0, 0, 20, 60, 42, 6, 0, 0, + 0, 0, 18, 58, 44, 8, 0, 0, + 0, 0, 16, 56, 46, 10, 0, 0, + 0, 0, 14, 54, 48, 12, 0, 0, + 0, 0, 12, 52, 52, 12, 0, 0, + 0, 0, 12, 48, 54, 14, 0, 0, + 0, 0, 10, 46, 56, 16, 0, 0, + 0, 0, 8, 44, 58, 18, 0, 0, + 0, 0, 6, 42, 60, 20, 0, 0, + 0, 0, 4, 40, 62, 22, 0, 0, + 0, 0, 4, 36, 62, 26, 0, 0, + 0, 0, 2, 34, 62, 30, 0, 0, + ]; + + /// + /// Gets the bilinear kernels for the sixteen subpixel phases. + /// + private static ReadOnlySpan Bilinear => + [ + 0, 0, 0, 128, 0, 0, 0, 0, + 0, 0, 0, 120, 8, 0, 0, 0, + 0, 0, 0, 112, 16, 0, 0, 0, + 0, 0, 0, 104, 24, 0, 0, 0, + 0, 0, 0, 96, 32, 0, 0, 0, + 0, 0, 0, 88, 40, 0, 0, 0, + 0, 0, 0, 80, 48, 0, 0, 0, + 0, 0, 0, 72, 56, 0, 0, 0, + 0, 0, 0, 64, 64, 0, 0, 0, + 0, 0, 0, 56, 72, 0, 0, 0, + 0, 0, 0, 48, 80, 0, 0, 0, + 0, 0, 0, 40, 88, 0, 0, 0, + 0, 0, 0, 32, 96, 0, 0, 0, + 0, 0, 0, 24, 104, 0, 0, 0, + 0, 0, 0, 16, 112, 0, 0, 0, + 0, 0, 0, 8, 120, 0, 0, 0, + ]; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.OneDimension.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.OneDimension.cs new file mode 100644 index 000000000..9bad7069c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.OneDimension.cs @@ -0,0 +1,541 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Provides SIMD kernels for horizontal-only and vertical-only single-reference filtering. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Filters an 8-bit block in sixteen-sample vectors. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + Vector128 initial) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + if (width < Vector128.Count) + { + // Four- and eight-sample AV1 blocks are smaller than one byte vector. Reference-plane padding makes + // the complete load readable, while the width-specific store leaves adjacent destination samples intact. + Convolve( + ref sourceRow, + tapStride, + 0, + ref coefficientBase, + tapCount, + initial, + out Vector128 result0, + out Vector128 result1, + out Vector128 result2, + out Vector128 result3); + + Round(ref result0, ref result1, ref result2, ref result3, firstRound, secondRound); + StorePartial(PackBytes(result0, result1, result2, result3), ref destinationRow, width); + continue; + } + + int vectorEnd = width - Vector128.Count; + for (; processedColumns <= vectorEnd; processedColumns += Vector128.Count) + { + Convolve( + ref sourceRow, + tapStride, + (nuint)processedColumns, + ref coefficientBase, + tapCount, + initial, + out Vector128 result0, + out Vector128 result1, + out Vector128 result2, + out Vector128 result3); + + Round(ref result0, ref result1, ref result2, ref result3, firstRound, secondRound); + PackBytes(result0, result1, result2, result3).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterDirectTail(ref sourceRow, ref destinationRow, processedColumns, width, tapStride, ref coefficientBase, tapCount, firstRound, secondRound); + } + } + + /// + /// Filters an 8-bit block in thirty-two-sample vectors. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + Vector256 initial) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + int vectorEnd = width - Vector256.Count; + + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector256.Count) + { + Convolve( + ref sourceRow, + tapStride, + (nuint)processedColumns, + ref coefficientBase, + tapCount, + initial, + out Vector256 result0, + out Vector256 result1, + out Vector256 result2, + out Vector256 result3); + + Round(ref result0, ref result1, ref result2, ref result3, firstRound, secondRound); + PackBytes(result0, result1, result2, result3).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterDirectTail(ref sourceRow, ref destinationRow, processedColumns, width, tapStride, ref coefficientBase, tapCount, firstRound, secondRound); + } + } + + /// + /// Filters an 8-bit block in sixty-four-sample vectors. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + Vector512 initial) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + int vectorEnd = width - Vector512.Count; + + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector512.Count) + { + Convolve( + ref sourceRow, + tapStride, + (nuint)processedColumns, + ref coefficientBase, + tapCount, + initial, + out Vector512 result0, + out Vector512 result1, + out Vector512 result2, + out Vector512 result3); + + Round(ref result0, ref result1, ref result2, ref result3, firstRound, secondRound); + PackBytes(result0, result1, result2, result3).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterDirectTail(ref sourceRow, ref destinationRow, processedColumns, width, tapStride, ref coefficientBase, tapCount, firstRound, secondRound); + } + } + + /// + /// Filters a high-bit-depth block in eight-sample vectors. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + int bitDepth, + Vector128 initial) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + ref ushort sourceRowUnsigned = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref short sourceRow = ref Unsafe.As(ref sourceRowUnsigned); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + if (width < Vector128.Count) + { + Convolve(ref sourceRow, tapStride, 0, ref coefficientBase, tapCount, initial, out Vector128 result0, out Vector128 result1); + Round(ref result0, ref result1, firstRound, secondRound); + + // The only legal AV1 width below eight is four samples, exactly the lower Vector64 half. + PackHighBitDepth(result0, result1, maximum).GetLower().StoreUnsafe(ref destinationRow); + continue; + } + + int vectorEnd = width - Vector128.Count; + for (; processedColumns <= vectorEnd; processedColumns += Vector128.Count) + { + Convolve(ref sourceRow, tapStride, (nuint)processedColumns, ref coefficientBase, tapCount, initial, out Vector128 result0, out Vector128 result1); + Round(ref result0, ref result1, firstRound, secondRound); + PackHighBitDepth(result0, result1, maximum).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterDirectTail(ref sourceRowUnsigned, ref destinationRow, processedColumns, width, tapStride, ref coefficientBase, tapCount, firstRound, secondRound, maximum); + } + } + + /// + /// Filters a high-bit-depth block in sixteen-sample vectors. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + int bitDepth, + Vector256 initial) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + int maximum = (1 << bitDepth) - 1; + int vectorEnd = width - Vector256.Count; + + for (int row = 0; row < height; row++) + { + ref ushort sourceRowUnsigned = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref short sourceRow = ref Unsafe.As(ref sourceRowUnsigned); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector256.Count) + { + Convolve(ref sourceRow, tapStride, (nuint)processedColumns, ref coefficientBase, tapCount, initial, out Vector256 result0, out Vector256 result1); + Round(ref result0, ref result1, firstRound, secondRound); + PackHighBitDepth(result0, result1, maximum).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterDirectTail(ref sourceRowUnsigned, ref destinationRow, processedColumns, width, tapStride, ref coefficientBase, tapCount, firstRound, secondRound, maximum); + } + } + + /// + /// Filters a high-bit-depth block in thirty-two-sample vectors. + /// + private static void FilterDirect( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + int bitDepth, + Vector512 initial) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + int maximum = (1 << bitDepth) - 1; + int vectorEnd = width - Vector512.Count; + + for (int row = 0; row < height; row++) + { + ref ushort sourceRowUnsigned = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref short sourceRow = ref Unsafe.As(ref sourceRowUnsigned); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector512.Count) + { + Convolve(ref sourceRow, tapStride, (nuint)processedColumns, ref coefficientBase, tapCount, initial, out Vector512 result0, out Vector512 result1); + Round(ref result0, ref result1, firstRound, secondRound); + PackHighBitDepth(result0, result1, maximum).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterDirectTail(ref sourceRowUnsigned, ref destinationRow, processedColumns, width, tapStride, ref coefficientBase, tapCount, firstRound, secondRound, maximum); + } + } + + /// + /// Applies the two direct-filter rounding stages to sixteen 8-bit results. + /// + private static void Round(ref Vector128 result0, ref Vector128 result1, ref Vector128 result2, ref Vector128 result3, int firstRound, int secondRound) + { + result0 = RoundPowerOfTwo(RoundPowerOfTwo(result0, firstRound), secondRound); + result1 = RoundPowerOfTwo(RoundPowerOfTwo(result1, firstRound), secondRound); + result2 = RoundPowerOfTwo(RoundPowerOfTwo(result2, firstRound), secondRound); + result3 = RoundPowerOfTwo(RoundPowerOfTwo(result3, firstRound), secondRound); + } + + /// + /// Applies the two direct-filter rounding stages to thirty-two 8-bit results. + /// + private static void Round(ref Vector256 result0, ref Vector256 result1, ref Vector256 result2, ref Vector256 result3, int firstRound, int secondRound) + { + result0 = RoundPowerOfTwo(RoundPowerOfTwo(result0, firstRound), secondRound); + result1 = RoundPowerOfTwo(RoundPowerOfTwo(result1, firstRound), secondRound); + result2 = RoundPowerOfTwo(RoundPowerOfTwo(result2, firstRound), secondRound); + result3 = RoundPowerOfTwo(RoundPowerOfTwo(result3, firstRound), secondRound); + } + + /// + /// Applies the two direct-filter rounding stages to sixty-four 8-bit results. + /// + private static void Round(ref Vector512 result0, ref Vector512 result1, ref Vector512 result2, ref Vector512 result3, int firstRound, int secondRound) + { + result0 = RoundPowerOfTwo(RoundPowerOfTwo(result0, firstRound), secondRound); + result1 = RoundPowerOfTwo(RoundPowerOfTwo(result1, firstRound), secondRound); + result2 = RoundPowerOfTwo(RoundPowerOfTwo(result2, firstRound), secondRound); + result3 = RoundPowerOfTwo(RoundPowerOfTwo(result3, firstRound), secondRound); + } + + /// + /// Applies the two direct-filter rounding stages to eight high-bit-depth results. + /// + private static void Round(ref Vector128 result0, ref Vector128 result1, int firstRound, int secondRound) + { + result0 = RoundPowerOfTwo(RoundPowerOfTwo(result0, firstRound), secondRound); + result1 = RoundPowerOfTwo(RoundPowerOfTwo(result1, firstRound), secondRound); + } + + /// + /// Applies the two direct-filter rounding stages to sixteen high-bit-depth results. + /// + private static void Round(ref Vector256 result0, ref Vector256 result1, int firstRound, int secondRound) + { + result0 = RoundPowerOfTwo(RoundPowerOfTwo(result0, firstRound), secondRound); + result1 = RoundPowerOfTwo(RoundPowerOfTwo(result1, firstRound), secondRound); + } + + /// + /// Applies the two direct-filter rounding stages to thirty-two high-bit-depth results. + /// + private static void Round(ref Vector512 result0, ref Vector512 result1, int firstRound, int secondRound) + { + result0 = RoundPowerOfTwo(RoundPowerOfTwo(result0, firstRound), secondRound); + result1 = RoundPowerOfTwo(RoundPowerOfTwo(result1, firstRound), secondRound); + } + + /// + /// Finishes an 8-bit row after its selected vector width. + /// + private static void FilterDirectTail( + ref byte source, + ref byte destination, + int firstColumn, + int width, + int tapStride, + ref short coefficients, + int tapCount, + int firstRound, + int secondRound) + { + for (int column = firstColumn; column < width; column++) + { + int sum = ConvolveScalar(ref Unsafe.Add(ref source, column), tapStride, ref coefficients, tapCount); + sum = RoundPowerOfTwo(sum, firstRound); + sum = RoundPowerOfTwo(sum, secondRound); + Unsafe.Add(ref destination, column) = (byte)Math.Clamp(sum, byte.MinValue, byte.MaxValue); + } + } + + /// + /// Finishes a high-bit-depth row after its selected vector width. + /// + private static void FilterDirectTail( + ref ushort source, + ref ushort destination, + int firstColumn, + int width, + int tapStride, + ref short coefficients, + int tapCount, + int firstRound, + int secondRound, + int maximum) + { + for (int column = firstColumn; column < width; column++) + { + int sum = ConvolveScalar(ref Unsafe.Add(ref source, column), tapStride, ref coefficients, tapCount); + sum = RoundPowerOfTwo(sum, firstRound); + sum = RoundPowerOfTwo(sum, secondRound); + Unsafe.Add(ref destination, column) = (ushort)Math.Clamp(sum, 0, maximum); + } + } + + /// + /// Stores the four- or eight-sample prefix of a sixteen-byte prediction vector. + /// + private static void StorePartial(Vector128 value, ref byte destination, int width) + { + if (width == 8) + { + value.GetLower().StoreUnsafe(ref destination); + } + else + { + Unsafe.As(ref destination) = value.AsUInt32().GetElement(0); + } + } + + /// + /// Applies a one-dimensional 8-bit filter without explicit hardware intrinsics. + /// + private static void FilterDirectScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + + for (int row = 0; row < height; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + int sum = ConvolveScalar(ref Unsafe.Add(ref sourceRow, column), tapStride, ref coefficientBase, tapCount); + sum = RoundPowerOfTwo(sum, firstRound); + + if (secondRound != 0) + { + sum = RoundPowerOfTwo(sum, secondRound); + } + + Unsafe.Add(ref destinationRow, column) = (byte)Math.Clamp(sum, byte.MinValue, byte.MaxValue); + } + } + } + + /// + /// Applies a one-dimensional high-bit-depth filter without explicit hardware intrinsics. + /// + private static void FilterDirectScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan coefficients, + int tapCount, + int sourceOffset, + int tapStride, + int firstRound, + int secondRound, + int bitDepth) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short coefficientBase = ref MemoryMarshal.GetReference(coefficients); + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + ref ushort sourceRow = ref Unsafe.Add(ref sourceBase, (row * sourceStride) + sourceOffset); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + int sum = ConvolveScalar(ref Unsafe.Add(ref sourceRow, column), tapStride, ref coefficientBase, tapCount); + sum = RoundPowerOfTwo(sum, firstRound); + + if (secondRound != 0) + { + sum = RoundPowerOfTwo(sum, secondRound); + } + + Unsafe.Add(ref destinationRow, column) = (ushort)Math.Clamp(sum, 0, maximum); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs new file mode 100644 index 000000000..a8739799d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs @@ -0,0 +1,446 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Defines interpolation operators and the generic traversal used by translational inter prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Supplies the normative Q7 coefficient kernel for one AV1 interpolation-filter family. + /// + /// + /// The closed operator type lets the JIT inline table selection into each horizontal and vertical filter pair. + /// Width reduction is selected once per block dimension rather than inside the sample loops. + /// + internal interface IAv1InterPredictorOperator + { + /// + /// Gets the eight coefficients for a one-sixteenth-sample phase. + /// + /// The fractional phase in the inclusive range zero through fifteen. + /// Indicates whether the coded block dimension is at most four samples. + /// The Q7 coefficients in increasing source-sample order. + public static abstract ReadOnlySpan GetCoefficients(int phase, bool useReducedFilter); + } + + /// + /// Selects an 8-bit vertical interpolation operator for a closed horizontal operator. + /// + /// The horizontal filter family. + private static void DispatchVertical( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span scratch, + bool scalarOnly) + where THorizontal : struct, IAv1InterPredictorOperator + { + switch (verticalFilter) + { + case Av1InterpolationFilter.Regular: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Smooth: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Sharp: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + default: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch, + scalarOnly); + + break; + } + } + + /// + /// Selects a high-bit-depth vertical interpolation operator for a closed horizontal operator. + /// + /// The horizontal filter family. + private static void DispatchVertical( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch, + bool scalarOnly) + where THorizontal : struct, IAv1InterPredictorOperator + { + switch (verticalFilter) + { + case Av1InterpolationFilter.Regular: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Smooth: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + case Av1InterpolationFilter.Sharp: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + default: + Predict( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + scalarOnly); + + break; + } + } + + /// + /// Executes one closed 8-bit interpolation-filter pair. + /// + /// The horizontal filter family. + /// The vertical filter family. + private static void Predict( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + int horizontalPhase, + int verticalPhase, + Span scratch, + bool scalarOnly) + where THorizontal : struct, IAv1InterPredictorOperator + where TVertical : struct, IAv1InterPredictorOperator + { + if (horizontalPhase == 0 && verticalPhase == 0) + { + Copy(source, sourceStride, sourceOrigin, destination, destinationStride, width, height, scalarOnly); + return; + } + + if (verticalPhase == 0) + { + ReadOnlySpan coefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + firstCoefficient - 3, + 1, + Round0Bits, + FilterBits - Round0Bits, + scalarOnly); + + return; + } + + if (horizontalPhase == 0) + { + ReadOnlySpan coefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + (firstCoefficient - 3) * sourceStride, + sourceStride, + FilterBits, + 0, + scalarOnly); + + return; + } + + ReadOnlySpan horizontalCoefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + ReadOnlySpan verticalCoefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(horizontalCoefficients, out int firstHorizontalCoefficient, out int horizontalTapCount); + GetEffectiveKernel(verticalCoefficients, out int firstVerticalCoefficient, out int verticalTapCount); + + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients[firstHorizontalCoefficient..], + horizontalTapCount, + firstHorizontalCoefficient - 3, + verticalCoefficients[firstVerticalCoefficient..], + verticalTapCount, + firstVerticalCoefficient - 3, + 8, + scratch, + scalarOnly); + } + + /// + /// Executes one closed high-bit-depth interpolation-filter pair. + /// + /// The horizontal filter family. + /// The vertical filter family. + private static void Predict( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch, + bool scalarOnly) + where THorizontal : struct, IAv1InterPredictorOperator + where TVertical : struct, IAv1InterPredictorOperator + { + if (horizontalPhase == 0 && verticalPhase == 0) + { + Copy(source, sourceStride, sourceOrigin, destination, destinationStride, width, height, scalarOnly); + return; + } + + // Twelve-bit samples require two additional first-pass rounding bits to keep libaom's signed intermediate + // within sixteen bits. The second pass gives those bits back, preserving a total Q14 shift. + int intermediateRange = bitDepth + FilterBits - Round0Bits + 2; + int round0 = Round0Bits + Math.Max(intermediateRange - 16, 0); + + if (verticalPhase == 0) + { + ReadOnlySpan coefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + firstCoefficient - 3, + 1, + round0, + FilterBits - round0, + bitDepth, + scalarOnly); + + return; + } + + if (horizontalPhase == 0) + { + ReadOnlySpan coefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + (firstCoefficient - 3) * sourceStride, + sourceStride, + FilterBits, + 0, + bitDepth, + scalarOnly); + + return; + } + + ReadOnlySpan horizontalCoefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + ReadOnlySpan verticalCoefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(horizontalCoefficients, out int firstHorizontalCoefficient, out int horizontalTapCount); + GetEffectiveKernel(verticalCoefficients, out int firstVerticalCoefficient, out int verticalTapCount); + + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients[firstHorizontalCoefficient..], + horizontalTapCount, + firstHorizontalCoefficient - 3, + verticalCoefficients[firstVerticalCoefficient..], + verticalTapCount, + firstVerticalCoefficient - 3, + bitDepth, + round0, + scratch, + scalarOnly); + } + + /// + /// Finds the centered nonzero portion of an eight-position interpolation kernel. + /// + /// The selected Q7 phase kernel. + /// Receives the first coefficient used by the effective kernel. + /// Receives the effective two-, four-, six-, or eight-tap length. + private static void GetEffectiveKernel(ReadOnlySpan coefficients, out int firstCoefficient, out int tapCount) + { + // This matches libaom's get_filter_tap decision. Reducing symmetric zero endpoints avoids source loads and + // multiply-adds while retaining the original tap-to-source alignment through firstCoefficient. + if (coefficients[0] != 0 || coefficients[7] != 0) + { + firstCoefficient = 0; + tapCount = 8; + } + else if (coefficients[1] != 0 || coefficients[6] != 0) + { + firstCoefficient = 1; + tapCount = 6; + } + else if (coefficients[2] != 0 || coefficients[5] != 0) + { + firstCoefficient = 2; + tapCount = 4; + } + else + { + firstCoefficient = 3; + tapCount = 2; + } + } + + /// + /// Selects one eight-coefficient phase from a flattened interpolation table. + /// + /// The sixteen consecutive phase kernels. + /// The selected one-sixteenth-sample phase. + /// The selected Q7 coefficient kernel. + private static ReadOnlySpan GetPhase(ReadOnlySpan table, int phase) => table.Slice(phase * FilterCoefficientCount, FilterCoefficientCount); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.RegularOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.RegularOperator.cs new file mode 100644 index 000000000..4eed3bad4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.RegularOperator.cs @@ -0,0 +1,20 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Defines regular interpolation for translational inter prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Selects regular interpolation coefficients. + /// + internal readonly struct RegularOperator : IAv1InterPredictorOperator + { + /// + public static ReadOnlySpan GetCoefficients(int phase, bool useReducedFilter) + => GetPhase(useReducedFilter ? RegularFourTap : RegularEightTap, phase); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.SharpOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.SharpOperator.cs new file mode 100644 index 000000000..b826a08a7 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.SharpOperator.cs @@ -0,0 +1,24 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Defines sharp interpolation for translational inter prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Selects sharp interpolation coefficients. + /// + internal readonly struct SharpOperator : IAv1InterPredictorOperator + { + /// + public static ReadOnlySpan GetCoefficients(int phase, bool useReducedFilter) + { + // AV1 defines sharp filtering on four-sample blocks to be identical to its reduced regular filter. + // Selecting that table here removes the distinction before the hot traversal is instantiated. + return GetPhase(useReducedFilter ? RegularFourTap : SharpEightTap, phase); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.SmoothOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.SmoothOperator.cs new file mode 100644 index 000000000..a3e883c6d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.SmoothOperator.cs @@ -0,0 +1,20 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Defines smooth interpolation for translational inter prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Selects smooth interpolation coefficients. + /// + internal readonly struct SmoothOperator : IAv1InterPredictorOperator + { + /// + public static ReadOnlySpan GetCoefficients(int phase, bool useReducedFilter) + => GetPhase(useReducedFilter ? SmoothFourTap : SmoothEightTap, phase); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.TwoDimensions.Byte.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.TwoDimensions.Byte.cs new file mode 100644 index 000000000..c03309350 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.TwoDimensions.Byte.cs @@ -0,0 +1,490 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Provides separable SIMD convolution for 8-bit single-reference prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Filters an 8-bit block in sixteen-sample vectors through caller-owned signed scratch. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + Vector128 initial) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + Vector128 horizontalInitial = initial + Vector128.Create(1 << (bitDepth + FilterBits - 1)); + + for (int row = 0; row < intermediateHeight; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + int processedColumns = 0; + + if (width < Vector128.Count) + { + Convolve( + ref sourceRow, + 1, + 0, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector128 result0, + out Vector128 result1, + out Vector128 result2, + out Vector128 result3); + + Av1IntraPredictorBase.Narrow(RoundPowerOfTwo(result0, round0), RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow); + Av1IntraPredictorBase.Narrow(RoundPowerOfTwo(result2, round0), RoundPowerOfTwo(result3, round0)).StoreUnsafe(ref scratchRow, (nuint)Vector128.Count); + continue; + } + + int vectorEnd = width - Vector128.Count; + for (; processedColumns <= vectorEnd; processedColumns += Vector128.Count) + { + Convolve( + ref sourceRow, + 1, + (nuint)processedColumns, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector128 result0, + out Vector128 result1, + out Vector128 result2, + out Vector128 result3); + + Av1IntraPredictorBase.Narrow(RoundPowerOfTwo(result0, round0), RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow, (nuint)processedColumns); + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result2, round0), + RoundPowerOfTwo(result3, round0)).StoreUnsafe( + ref scratchRow, + (nuint)(processedColumns + Vector128.Count)); + } + + FilterHorizontalTail(ref sourceRow, ref scratchRow, processedColumns, width, ref horizontalCoefficientBase, horizontalTapCount, bitDepth, round0); + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + Vector128 verticalInitial = initial + Vector128.Create(1 << offsetBits); + Vector128 roundOffset = Vector128.Create((1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1))); + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + if (width < Vector128.Count) + { + Convolve( + ref scratchRow, + scratchStride, + 0, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector128 result0, + out Vector128 result1); + + Convolve( + ref Unsafe.Add(ref scratchRow, Vector128.Count), + scratchStride, + 0, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector128 result2, + out Vector128 result3); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + result2 = RoundPowerOfTwo(result2, round1) - roundOffset; + result3 = RoundPowerOfTwo(result3, round1) - roundOffset; + StorePartial(PackBytes(result0, result1, result2, result3), ref destinationRow, width); + continue; + } + + int vectorEnd = width - Vector128.Count; + for (; processedColumns <= vectorEnd; processedColumns += Vector128.Count) + { + Convolve( + ref scratchRow, + scratchStride, + (nuint)processedColumns, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector128 result0, + out Vector128 result1); + + Convolve( + ref scratchRow, + scratchStride, + (nuint)(processedColumns + Vector128.Count), + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector128 result2, + out Vector128 result3); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + result2 = RoundPowerOfTwo(result2, round1) - roundOffset; + result3 = RoundPowerOfTwo(result3, round1) - roundOffset; + PackBytes(result0, result1, result2, result3).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterVerticalTail(ref scratchRow, ref destinationRow, processedColumns, width, scratchStride, ref verticalCoefficientBase, verticalTapCount, bitDepth, round0); + } + } + + /// + /// Filters an 8-bit block in thirty-two-sample vectors through caller-owned signed scratch. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + Vector256 initial) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + Vector256 horizontalInitial = initial + Vector256.Create(1 << (bitDepth + FilterBits - 1)); + int vectorEnd = width - Vector256.Count; + + for (int row = 0; row < intermediateHeight; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector256.Count) + { + Convolve( + ref sourceRow, + 1, + (nuint)processedColumns, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector256 result0, + out Vector256 result1, + out Vector256 result2, + out Vector256 result3); + + Av1IntraPredictorBase.Narrow(RoundPowerOfTwo(result0, round0), RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow, (nuint)processedColumns); + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result2, round0), + RoundPowerOfTwo(result3, round0)).StoreUnsafe( + ref scratchRow, + (nuint)(processedColumns + Vector256.Count)); + } + + FilterHorizontalTail(ref sourceRow, ref scratchRow, processedColumns, width, ref horizontalCoefficientBase, horizontalTapCount, bitDepth, round0); + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + Vector256 verticalInitial = initial + Vector256.Create(1 << offsetBits); + Vector256 roundOffset = Vector256.Create((1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1))); + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector256.Count) + { + Convolve( + ref scratchRow, + scratchStride, + (nuint)processedColumns, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector256 result0, + out Vector256 result1); + + Convolve( + ref scratchRow, + scratchStride, + (nuint)(processedColumns + Vector256.Count), + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector256 result2, + out Vector256 result3); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + result2 = RoundPowerOfTwo(result2, round1) - roundOffset; + result3 = RoundPowerOfTwo(result3, round1) - roundOffset; + PackBytes(result0, result1, result2, result3).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterVerticalTail(ref scratchRow, ref destinationRow, processedColumns, width, scratchStride, ref verticalCoefficientBase, verticalTapCount, bitDepth, round0); + } + } + + /// + /// Filters an 8-bit block in sixty-four-sample vectors through caller-owned signed scratch. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + Vector512 initial) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + Vector512 horizontalInitial = initial + Vector512.Create(1 << (bitDepth + FilterBits - 1)); + int vectorEnd = width - Vector512.Count; + + for (int row = 0; row < intermediateHeight; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector512.Count) + { + Convolve( + ref sourceRow, + 1, + (nuint)processedColumns, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector512 result0, + out Vector512 result1, + out Vector512 result2, + out Vector512 result3); + + Av1IntraPredictorBase.Narrow(RoundPowerOfTwo(result0, round0), RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow, (nuint)processedColumns); + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result2, round0), + RoundPowerOfTwo(result3, round0)).StoreUnsafe( + ref scratchRow, + (nuint)(processedColumns + Vector512.Count)); + } + + FilterHorizontalTail(ref sourceRow, ref scratchRow, processedColumns, width, ref horizontalCoefficientBase, horizontalTapCount, bitDepth, round0); + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + Vector512 verticalInitial = initial + Vector512.Create(1 << offsetBits); + Vector512 roundOffset = Vector512.Create((1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1))); + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector512.Count) + { + Convolve( + ref scratchRow, + scratchStride, + (nuint)processedColumns, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector512 result0, + out Vector512 result1); + + Convolve( + ref scratchRow, + scratchStride, + (nuint)(processedColumns + Vector512.Count), + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector512 result2, + out Vector512 result3); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + result2 = RoundPowerOfTwo(result2, round1) - roundOffset; + result3 = RoundPowerOfTwo(result3, round1) - roundOffset; + PackBytes(result0, result1, result2, result3).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterVerticalTail(ref scratchRow, ref destinationRow, processedColumns, width, scratchStride, ref verticalCoefficientBase, verticalTapCount, bitDepth, round0); + } + } + + /// + /// Finishes an 8-bit horizontal intermediate row after its selected vector width. + /// + private static void FilterHorizontalTail(ref byte source, ref short scratch, int firstColumn, int width, ref short coefficients, int tapCount, int bitDepth, int round0) + { + int horizontalBias = 1 << (bitDepth + FilterBits - 1); + for (int column = firstColumn; column < width; column++) + { + int sum = horizontalBias + ConvolveScalar(ref Unsafe.Add(ref source, column), 1, ref coefficients, tapCount); + Unsafe.Add(ref scratch, column) = (short)RoundPowerOfTwo(sum, round0); + } + } + + /// + /// Finishes an 8-bit vertical output row after its selected vector width. + /// + private static void FilterVerticalTail( + ref short scratch, + ref byte destination, + int firstColumn, + int width, + int scratchStride, + ref short coefficients, + int tapCount, + int bitDepth, + int round0) + { + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + int verticalBias = 1 << offsetBits; + int roundOffset = (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1)); + + for (int column = firstColumn; column < width; column++) + { + int sum = verticalBias + ConvolveScalar(ref Unsafe.Add(ref scratch, column), scratchStride, ref coefficients, tapCount); + int result = RoundPowerOfTwo(sum, round1) - roundOffset; + Unsafe.Add(ref destination, column) = (byte)Math.Clamp(result, byte.MinValue, byte.MaxValue); + } + } + + /// + /// Applies separable two-dimensional filtering to an 8-bit block without explicit hardware intrinsics. + /// + private static void Filter2DScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch) + { + ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + int horizontalBias = 1 << (bitDepth + FilterBits - 1); + + // The first intermediate row corresponds to the uppermost vertical tap. Horizontal filtering therefore starts + // above the nominal source origin and writes one row for every vertical-tap position needed by the final pass. + for (int row = 0; row < intermediateHeight; row++) + { + ref byte sourceRow = ref Unsafe.Add(ref sourceBase, ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + + for (int column = 0; column < width; column++) + { + int sum = horizontalBias + ConvolveScalar(ref Unsafe.Add(ref sourceRow, column), 1, ref horizontalCoefficientBase, horizontalTapCount); + Unsafe.Add(ref scratchRow, column) = (short)RoundPowerOfTwo(sum, round0); + } + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + int verticalBias = 1 << offsetBits; + int roundOffset = (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1)); + + // The biased first pass keeps every intermediate nonnegative and representable by a signed 16-bit lane. + // Removing both bias terms after the vertical Q7 filter reproduces libaom's single-reference rounding exactly. + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref byte destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + int sum = verticalBias + ConvolveScalar(ref Unsafe.Add(ref scratchRow, column), scratchStride, ref verticalCoefficientBase, verticalTapCount); + int result = RoundPowerOfTwo(sum, round1) - roundOffset; + Unsafe.Add(ref destinationRow, column) = (byte)Math.Clamp(result, byte.MinValue, byte.MaxValue); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.TwoDimensions.UInt16.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.TwoDimensions.UInt16.cs new file mode 100644 index 000000000..9c3fde100 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.TwoDimensions.UInt16.cs @@ -0,0 +1,519 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Provides separable SIMD convolution for unsigned 16-bit single-reference prediction. +/// +internal static partial class Av1InterPredictor +{ + /// + /// Filters a high-bit-depth block in eight-sample vectors through caller-owned signed scratch. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + Vector128 initial) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + Vector128 horizontalInitial = initial + Vector128.Create(1 << (bitDepth + FilterBits - 1)); + + for (int row = 0; row < intermediateHeight; row++) + { + ref ushort sourceRowUnsigned = ref Unsafe.Add( + ref sourceBase, + ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + + ref short sourceRow = ref Unsafe.As(ref sourceRowUnsigned); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + int processedColumns = 0; + + if (width < Vector128.Count) + { + // AV1's only legal width below eight is four. The reference plane is padded for the full source load, + // and the minimum scratch stride preserves all eight intermediate lanes needed by the vertical pass. + Convolve( + ref sourceRow, + 1, + 0, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector128 result0, + out Vector128 result1); + + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result0, round0), + RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow); + + continue; + } + + int vectorEnd = width - Vector128.Count; + for (; processedColumns <= vectorEnd; processedColumns += Vector128.Count) + { + Convolve( + ref sourceRow, + 1, + (nuint)processedColumns, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector128 result0, + out Vector128 result1); + + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result0, round0), + RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow, (nuint)processedColumns); + } + + FilterHorizontalTail( + ref sourceRowUnsigned, + ref scratchRow, + processedColumns, + width, + ref horizontalCoefficientBase, + horizontalTapCount, + bitDepth, + round0); + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + Vector128 verticalInitial = initial + Vector128.Create(1 << offsetBits); + Vector128 roundOffset = Vector128.Create( + (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1))); + + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + if (width < Vector128.Count) + { + Convolve( + ref scratchRow, + scratchStride, + 0, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector128 result0, + out Vector128 result1); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + + // Four high-bit-depth samples occupy exactly the lower half of the packed vector. + PackHighBitDepth(result0, result1, maximum).GetLower().StoreUnsafe(ref destinationRow); + continue; + } + + int vectorEnd = width - Vector128.Count; + for (; processedColumns <= vectorEnd; processedColumns += Vector128.Count) + { + Convolve( + ref scratchRow, + scratchStride, + (nuint)processedColumns, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector128 result0, + out Vector128 result1); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + PackHighBitDepth(result0, result1, maximum).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterVerticalTail( + ref scratchRow, + ref destinationRow, + processedColumns, + width, + scratchStride, + ref verticalCoefficientBase, + verticalTapCount, + bitDepth, + round0); + } + } + + /// + /// Filters a high-bit-depth block in sixteen-sample vectors through caller-owned signed scratch. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + Vector256 initial) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + Vector256 horizontalInitial = initial + Vector256.Create(1 << (bitDepth + FilterBits - 1)); + int vectorEnd = width - Vector256.Count; + + for (int row = 0; row < intermediateHeight; row++) + { + ref ushort sourceRowUnsigned = ref Unsafe.Add( + ref sourceBase, + ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + + ref short sourceRow = ref Unsafe.As(ref sourceRowUnsigned); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector256.Count) + { + Convolve( + ref sourceRow, + 1, + (nuint)processedColumns, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector256 result0, + out Vector256 result1); + + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result0, round0), + RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow, (nuint)processedColumns); + } + + FilterHorizontalTail( + ref sourceRowUnsigned, + ref scratchRow, + processedColumns, + width, + ref horizontalCoefficientBase, + horizontalTapCount, + bitDepth, + round0); + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + Vector256 verticalInitial = initial + Vector256.Create(1 << offsetBits); + Vector256 roundOffset = Vector256.Create( + (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1))); + + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector256.Count) + { + Convolve( + ref scratchRow, + scratchStride, + (nuint)processedColumns, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector256 result0, + out Vector256 result1); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + PackHighBitDepth(result0, result1, maximum).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterVerticalTail( + ref scratchRow, + ref destinationRow, + processedColumns, + width, + scratchStride, + ref verticalCoefficientBase, + verticalTapCount, + bitDepth, + round0); + } + } + + /// + /// Filters a high-bit-depth block in thirty-two-sample vectors through caller-owned signed scratch. + /// + private static void Filter2D( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch, + Vector512 initial) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + Vector512 horizontalInitial = initial + Vector512.Create(1 << (bitDepth + FilterBits - 1)); + int vectorEnd = width - Vector512.Count; + + for (int row = 0; row < intermediateHeight; row++) + { + ref ushort sourceRowUnsigned = ref Unsafe.Add( + ref sourceBase, + ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + + ref short sourceRow = ref Unsafe.As(ref sourceRowUnsigned); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector512.Count) + { + Convolve( + ref sourceRow, + 1, + (nuint)processedColumns, + ref horizontalCoefficientBase, + horizontalTapCount, + horizontalInitial, + out Vector512 result0, + out Vector512 result1); + + Av1IntraPredictorBase.Narrow( + RoundPowerOfTwo(result0, round0), + RoundPowerOfTwo(result1, round0)).StoreUnsafe(ref scratchRow, (nuint)processedColumns); + } + + FilterHorizontalTail( + ref sourceRowUnsigned, + ref scratchRow, + processedColumns, + width, + ref horizontalCoefficientBase, + horizontalTapCount, + bitDepth, + round0); + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + Vector512 verticalInitial = initial + Vector512.Create(1 << offsetBits); + Vector512 roundOffset = Vector512.Create( + (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1))); + + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + int processedColumns = 0; + + for (; processedColumns <= vectorEnd; processedColumns += Vector512.Count) + { + Convolve( + ref scratchRow, + scratchStride, + (nuint)processedColumns, + ref verticalCoefficientBase, + verticalTapCount, + verticalInitial, + out Vector512 result0, + out Vector512 result1); + + result0 = RoundPowerOfTwo(result0, round1) - roundOffset; + result1 = RoundPowerOfTwo(result1, round1) - roundOffset; + PackHighBitDepth(result0, result1, maximum).StoreUnsafe(ref destinationRow, (nuint)processedColumns); + } + + FilterVerticalTail( + ref scratchRow, + ref destinationRow, + processedColumns, + width, + scratchStride, + ref verticalCoefficientBase, + verticalTapCount, + bitDepth, + round0); + } + } + + /// + /// Finishes a high-bit-depth horizontal intermediate row after its selected vector width. + /// + private static void FilterHorizontalTail( + ref ushort source, + ref short scratch, + int firstColumn, + int width, + ref short coefficients, + int tapCount, + int bitDepth, + int round0) + { + int horizontalBias = 1 << (bitDepth + FilterBits - 1); + for (int column = firstColumn; column < width; column++) + { + int sum = horizontalBias + ConvolveScalar( + ref Unsafe.Add(ref source, column), + 1, + ref coefficients, + tapCount); + + Unsafe.Add(ref scratch, column) = (short)RoundPowerOfTwo(sum, round0); + } + } + + /// + /// Finishes a high-bit-depth vertical output row after its selected vector width. + /// + private static void FilterVerticalTail( + ref short scratch, + ref ushort destination, + int firstColumn, + int width, + int scratchStride, + ref short coefficients, + int tapCount, + int bitDepth, + int round0) + { + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + int verticalBias = 1 << offsetBits; + int roundOffset = (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1)); + int maximum = (1 << bitDepth) - 1; + + for (int column = firstColumn; column < width; column++) + { + int sum = verticalBias + ConvolveScalar( + ref Unsafe.Add(ref scratch, column), + scratchStride, + ref coefficients, + tapCount); + + int result = RoundPowerOfTwo(sum, round1) - roundOffset; + Unsafe.Add(ref destination, column) = (ushort)Math.Clamp(result, 0, maximum); + } + } + + /// + /// Applies separable two-dimensional filtering to a high-bit-depth block without explicit hardware intrinsics. + /// + private static void Filter2DScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + ReadOnlySpan horizontalCoefficients, + int horizontalTapCount, + int horizontalSourceOffset, + ReadOnlySpan verticalCoefficients, + int verticalTapCount, + int verticalSourceOffset, + int bitDepth, + int round0, + Span scratch) + { + ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); + ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); + ref short scratchBase = ref MemoryMarshal.GetReference(scratch); + ref short horizontalCoefficientBase = ref MemoryMarshal.GetReference(horizontalCoefficients); + ref short verticalCoefficientBase = ref MemoryMarshal.GetReference(verticalCoefficients); + int scratchStride = Math.Max(width, MinimumScratchStride); + int intermediateHeight = height + verticalTapCount - 1; + int horizontalBias = 1 << (bitDepth + FilterBits - 1); + + for (int row = 0; row < intermediateHeight; row++) + { + ref ushort sourceRow = ref Unsafe.Add(ref sourceBase, ((row + verticalSourceOffset) * sourceStride) + horizontalSourceOffset); + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + + for (int column = 0; column < width; column++) + { + int sum = horizontalBias + ConvolveScalar(ref Unsafe.Add(ref sourceRow, column), 1, ref horizontalCoefficientBase, horizontalTapCount); + Unsafe.Add(ref scratchRow, column) = (short)RoundPowerOfTwo(sum, round0); + } + } + + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + int verticalBias = 1 << offsetBits; + int roundOffset = (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1)); + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + ref short scratchRow = ref Unsafe.Add(ref scratchBase, row * scratchStride); + ref ushort destinationRow = ref Unsafe.Add(ref destinationBase, row * destinationStride); + + for (int column = 0; column < width; column++) + { + int sum = verticalBias + ConvolveScalar(ref Unsafe.Add(ref scratchRow, column), scratchStride, ref verticalCoefficientBase, verticalTapCount); + int result = RoundPowerOfTwo(sum, round1) - roundOffset; + Unsafe.Add(ref destinationRow, column) = (ushort)Math.Clamp(result, 0, maximum); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs new file mode 100644 index 000000000..8c43adbec --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs @@ -0,0 +1,238 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Reconstructs single-reference translational AV1 inter-prediction blocks. +/// +/// +/// +/// sourceOrigin identifies the integer sample selected by motion-vector scaling within the complete +/// padded reference plane. A filtered axis can consume three samples before the block and four samples after it. Byte +/// rows narrower than sixteen samples and 16-bit rows narrower than eight samples must additionally permit a complete +/// 128-bit source load at every selected tap. The frame prediction border provides this storage; no destination padding +/// is required. +/// +/// +/// Two-dimensional filtering uses caller-owned scratch so block reconstruction does not allocate. The scratch span must +/// contain at least elements when both phases are nonzero and may be empty for +/// copy or one-dimensional filtering. +/// +/// +internal static partial class Av1InterPredictor +{ + /// + /// The number of fractional bits in each interpolation coefficient. + /// + private const int FilterBits = 7; + + /// + /// The normal first-round shift used by libaom single-reference convolution. + /// + private const int Round0Bits = 3; + + /// + /// The maximum number of source rows added by an eight-tap vertical filter. + /// + private const int MaximumExtraRows = FilterCoefficientCount - 1; + + /// + /// The minimum scratch stride that lets a 128-bit byte kernel handle four- and eight-sample blocks. + /// + private const int MinimumScratchStride = 16; + + /// + /// Gets the maximum number of signed 16-bit elements required for one two-dimensional prediction block. + /// + /// The prediction width in samples. + /// The prediction height in samples. + /// The scratch capacity required by either sample-storage overload. + public static int GetScratchLength(int width, int height) => Math.Max(width, MinimumScratchStride) * (height + MaximumExtraRows); + + /// + /// Reconstructs an 8-bit translational prediction using the widest supported SIMD kernel. + /// + /// The complete padded reference plane containing every source sample used by the block. + /// The distance between reference rows in samples. + /// The nonnegative, zero-based index of the integer-position source sample within . + /// The prediction block destination. + /// The distance between destination rows in samples. + /// The prediction width in samples. + /// The prediction height in samples. + /// The horizontal interpolation filter. + /// The vertical interpolation filter. + /// The horizontal phase in one-sixteenth-sample units. + /// The vertical phase in one-sixteenth-sample units. + /// + /// Caller-owned signed intermediate storage sized by when both phases are nonzero. + /// + public static void Predict( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span scratch) + => Dispatch( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalFilter, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch, + false); + + /// + /// Reconstructs an 8-, 10-, or 12-bit translational prediction using the widest supported SIMD kernel. + /// + /// The complete padded reference plane containing every source sample used by the block. + /// The distance between reference rows in samples. + /// The nonnegative, zero-based index of the integer-position source sample within . + /// The prediction block destination. + /// The distance between destination rows in samples. + /// The prediction width in samples. + /// The prediction height in samples. + /// The horizontal interpolation filter. + /// The vertical interpolation filter. + /// The horizontal phase in one-sixteenth-sample units. + /// The vertical phase in one-sixteenth-sample units. + /// The decoded sample precision: 8, 10, or 12 bits. + /// + /// Caller-owned signed intermediate storage sized by when both phases are nonzero. + /// + public static void Predict( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch) + => Dispatch( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalFilter, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + false); + + /// + /// Reconstructs an 8-bit translational prediction without explicit hardware intrinsics. + /// + /// The complete padded reference plane containing every source sample used by the block. + /// The distance between reference rows in samples. + /// The nonnegative, zero-based index of the integer-position source sample within . + /// The prediction block destination. + /// The distance between destination rows in samples. + /// The prediction width in samples. + /// The prediction height in samples. + /// The horizontal interpolation filter. + /// The vertical interpolation filter. + /// The horizontal phase in one-sixteenth-sample units. + /// The vertical phase in one-sixteenth-sample units. + /// + /// Caller-owned signed intermediate storage sized by when both phases are nonzero. + /// + public static void PredictScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span scratch) + => Dispatch( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalFilter, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch, + true); + + /// + /// Reconstructs an 8-, 10-, or 12-bit translational prediction without explicit hardware intrinsics. + /// + /// The complete padded reference plane containing every source sample used by the block. + /// The distance between reference rows in samples. + /// The nonnegative, zero-based index of the integer-position source sample within . + /// The prediction block destination. + /// The distance between destination rows in samples. + /// The prediction width in samples. + /// The prediction height in samples. + /// The horizontal interpolation filter. + /// The vertical interpolation filter. + /// The horizontal phase in one-sixteenth-sample units. + /// The vertical phase in one-sixteenth-sample units. + /// The decoded sample precision: 8, 10, or 12 bits. + /// + /// Caller-owned signed intermediate storage sized by when both phases are nonzero. + /// + public static void PredictScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch) + => Dispatch( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalFilter, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch, + true); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs new file mode 100644 index 000000000..b2bc9c5fa --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs @@ -0,0 +1,30 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + +/// +/// Identifies an AV1 interpolation filter used for translational inter prediction. +/// +internal enum Av1InterpolationFilter +{ + /// + /// The regular interpolation-filter family. + /// + Regular, + + /// + /// The smooth interpolation-filter family. + /// + Smooth, + + /// + /// The sharp interpolation-filter family. + /// + Sharp, + + /// + /// The bilinear interpolation-filter family. + /// + Bilinear, +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterPredictorTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterPredictorTests.cs new file mode 100644 index 000000000..0659680d3 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterPredictorTests.cs @@ -0,0 +1,885 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; +using SixLabors.ImageSharp.Tests.TestUtilities; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies AV1 translational inter prediction against an independent implementation of the normative fixed-point convolution rules. +/// +[Trait("Format", "Avif")] +public class Av1InterPredictorTests +{ + /// + /// The number of fractional coefficient bits in AV1 interpolation kernels. + /// + private const int FilterBits = 7; + + /// + /// The default first convolution shift used for 8- and 10-bit predictions. + /// + private const int Round0Bits = 3; + + /// + /// The number of stored coefficient positions in every tested interpolation kernel. + /// + private const int FilterTapCount = 8; + + /// + /// The number of coefficient positions preceding the integer-position sample. + /// + private const int FilterCenterOffset = 3; + + /// + /// The source samples retained before the integer-position column. + /// + private const int SourceLeftPadding = 3; + + /// + /// The source samples retained after each active row for a complete 512-bit byte load. + /// + private const int SourceRightPadding = 64; + + /// + /// The source rows retained before the integer-position row. + /// + private const int SourceTopPadding = 3; + + /// + /// The source rows retained after the prediction block. + /// + private const int SourceBottomPadding = 4; + + /// + /// The guarded destination elements preceding the first active row. + /// + private const int DestinationPrefix = 11; + + /// + /// The guarded destination elements following the final padded row. + /// + private const int DestinationSuffix = 17; + + /// + /// The guarded destination elements following each active row. + /// + private const int DestinationRowPadding = 13; + + /// + /// The non-image value stored in every guarded 8-bit destination element. + /// + private const byte ByteDestinationSentinel = 0xD3; + + /// + /// The non-image value stored in every guarded ushort destination element. + /// + private const ushort HighBitDepthDestinationSentinel = 0xDEAD; + + /// + /// Exercises the native vector width, the 256-bit path, the 128-bit path, and the complete scalar fallback. + /// + /// + /// Disabling AVX also disables AVX2 and leaves the x86 128-bit vector tier enabled, which is the established + /// configuration used by the other AV1 SIMD tests. + /// + private const HwIntrinsics PredictorConfigurations = + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + + /// + /// Verifies exact 8-bit copy and convolution output, scalar tails, and untouched destination padding under every SIMD configuration. + /// + [Fact] + public void BytePredictionMatchesLibaomOracleAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateBytePredictions, PredictorConfigurations); + + /// + /// Verifies exact 8-, 10-, and 12-bit ushort output, scalar tails, and untouched destination padding under every SIMD configuration. + /// + [Fact] + public void HighBitDepthPredictionMatchesLibaomOracleAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateHighBitDepthPredictions, PredictorConfigurations); + + /// + /// Applies every byte prediction scenario to the SIMD-first and explicitly scalar entry points. + /// + private static void ValidateBytePredictions() + { + foreach (PredictionCase testCase in CreatePredictionCases()) + { + byte[] source = CreateByteSource(testCase, out int sourceStride, out int sourceOrigin); + int destinationStride = testCase.Width + DestinationRowPadding; + byte[] expected = CreateByteDestination(testCase, destinationStride); + byte[] actual = (byte[])expected.Clone(); + byte[] scalar = (byte[])expected.Clone(); + short[] simdScratch = CreateScratch(testCase); + short[] scalarScratch = CreateScratch(testCase); + + ApplyReference(source, sourceStride, sourceOrigin, expected, DestinationPrefix, destinationStride, testCase, 8); + + Av1InterPredictor.Predict( + source, sourceStride, sourceOrigin, actual.AsSpan(DestinationPrefix), destinationStride, testCase.Width, testCase.Height, + testCase.HorizontalFilter, testCase.VerticalFilter, testCase.HorizontalPhase, testCase.VerticalPhase, simdScratch); + + Av1InterPredictor.PredictScalar( + source, sourceStride, sourceOrigin, scalar.AsSpan(DestinationPrefix), destinationStride, testCase.Width, testCase.Height, + testCase.HorizontalFilter, testCase.VerticalFilter, testCase.HorizontalPhase, testCase.VerticalPhase, scalarScratch); + + AssertEqual(expected, actual, testCase, "SIMD-first byte"); + AssertEqual(expected, scalar, testCase, "scalar byte"); + } + } + + /// + /// Applies every ushort prediction scenario at each supported sample precision to the SIMD-first and scalar entry points. + /// + private static void ValidateHighBitDepthPredictions() + { + int[] bitDepths = [8, 10, 12]; + foreach (int bitDepth in bitDepths) + { + foreach (PredictionCase testCase in CreatePredictionCases()) + { + ushort[] source = CreateHighBitDepthSource(testCase, bitDepth, out int sourceStride, out int sourceOrigin); + int destinationStride = testCase.Width + DestinationRowPadding; + ushort[] expected = CreateHighBitDepthDestination(testCase, destinationStride); + ushort[] actual = (ushort[])expected.Clone(); + ushort[] scalar = (ushort[])expected.Clone(); + short[] simdScratch = CreateScratch(testCase); + short[] scalarScratch = CreateScratch(testCase); + + ApplyReference(source, sourceStride, sourceOrigin, expected, DestinationPrefix, destinationStride, testCase, bitDepth); + + Av1InterPredictor.Predict( + source, sourceStride, sourceOrigin, actual.AsSpan(DestinationPrefix), destinationStride, testCase.Width, testCase.Height, + testCase.HorizontalFilter, testCase.VerticalFilter, testCase.HorizontalPhase, testCase.VerticalPhase, bitDepth, simdScratch); + + Av1InterPredictor.PredictScalar( + source, sourceStride, sourceOrigin, scalar.AsSpan(DestinationPrefix), destinationStride, testCase.Width, testCase.Height, + testCase.HorizontalFilter, testCase.VerticalFilter, testCase.HorizontalPhase, testCase.VerticalPhase, bitDepth, scalarScratch); + + AssertEqual(expected, actual, testCase, $"SIMD-first {bitDepth}-bit ushort"); + AssertEqual(expected, scalar, testCase, $"scalar {bitDepth}-bit ushort"); + } + } + } + + /// + /// Creates the named operation matrix covering copy, each one-dimensional direction, separable filtering, reduced kernels, and vector tails. + /// + /// The prediction scenarios. + private static PredictionCase[] CreatePredictionCases() => + [ + new("copy-wide-tail", 68, 8, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Sharp, 0, 0), + new("regular-horizontal-wide-tail", 68, 8, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Regular, 1, 0), + new("smooth-horizontal-256-tail", 36, 8, Av1InterpolationFilter.Smooth, Av1InterpolationFilter.Regular, 7, 0), + new("sharp-horizontal-128-tail", 20, 8, Av1InterpolationFilter.Sharp, Av1InterpolationFilter.Regular, 8, 0), + new("bilinear-horizontal", 8, 8, Av1InterpolationFilter.Bilinear, Av1InterpolationFilter.Regular, 15, 0), + new("regular-horizontal-reduced", 4, 8, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Regular, 3, 0), + new("smooth-horizontal-reduced", 4, 8, Av1InterpolationFilter.Smooth, Av1InterpolationFilter.Regular, 13, 0), + new("sharp-horizontal-reduced", 4, 8, Av1InterpolationFilter.Sharp, Av1InterpolationFilter.Regular, 5, 0), + new("regular-vertical-reduced", 68, 4, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Regular, 0, 3), + new("smooth-vertical-reduced", 36, 4, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Smooth, 0, 13), + new("sharp-vertical-reduced", 20, 4, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Sharp, 0, 5), + new("bilinear-vertical", 8, 8, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Bilinear, 0, 8), + new("regular-smooth-two-dimensional", 68, 8, Av1InterpolationFilter.Regular, Av1InterpolationFilter.Smooth, 1, 15), + new("sharp-bilinear-two-dimensional", 36, 8, Av1InterpolationFilter.Sharp, Av1InterpolationFilter.Bilinear, 8, 3), + new("smooth-sharp-reduced-two-dimensional", 4, 4, Av1InterpolationFilter.Smooth, Av1InterpolationFilter.Sharp, 7, 13), + new("bilinear-regular-small-height", 20, 4, Av1InterpolationFilter.Bilinear, Av1InterpolationFilter.Regular, 11, 5), + new("sharp-smooth-small-width", 4, 8, Av1InterpolationFilter.Sharp, Av1InterpolationFilter.Smooth, 5, 7) + ]; + + /// + /// Creates an 8-bit padded reference plane and returns the integer-position source origin within that plane. + /// + /// The prediction geometry used to size the plane. + /// Receives the padded source-row stride. + /// Receives the integer-position sample index. + /// The complete padded source plane. + private static byte[] CreateByteSource(PredictionCase testCase, out int sourceStride, out int sourceOrigin) + { + sourceStride = SourceLeftPadding + testCase.Width + SourceRightPadding; + int sourceHeight = SourceTopPadding + testCase.Height + SourceBottomPadding; + byte[] source = new byte[sourceStride * sourceHeight]; + + for (int row = 0; row < sourceHeight; row++) + { + for (int column = 0; column < sourceStride; column++) + { + // Distinct row, column, and cross-term multipliers prevent a wrong stride or tap direction from + // producing the same arithmetic progression as the correctly addressed source window. + source[(row * sourceStride) + column] = (byte)(((row * 59) + (column * 37) + (row * column * 11) + 17) & byte.MaxValue); + } + } + + sourceOrigin = (SourceTopPadding * sourceStride) + SourceLeftPadding; + return source; + } + + /// + /// Creates a padded high-bit-depth reference plane spanning the legal range for the requested precision. + /// + /// The prediction geometry used to size the plane. + /// The decoded sample precision. + /// Receives the padded source-row stride. + /// Receives the integer-position sample index. + /// The complete padded source plane. + private static ushort[] CreateHighBitDepthSource(PredictionCase testCase, int bitDepth, out int sourceStride, out int sourceOrigin) + { + sourceStride = SourceLeftPadding + testCase.Width + SourceRightPadding; + int sourceHeight = SourceTopPadding + testCase.Height + SourceBottomPadding; + int maximum = (1 << bitDepth) - 1; + ushort[] source = new ushort[sourceStride * sourceHeight]; + + for (int row = 0; row < sourceHeight; row++) + { + for (int column = 0; column < sourceStride; column++) + { + // The high-bit-depth pattern uses different coprime multipliers and spans the complete requested + // range, exercising negative-lobe clipping as well as low and high sample values. + source[(row * sourceStride) + column] = (ushort)(((row * 977) + (column * 353) + (row * column * 29) + 101) & maximum); + } + } + + sourceOrigin = (SourceTopPadding * sourceStride) + SourceLeftPadding; + return source; + } + + /// + /// Creates an 8-bit destination whose prefix, row padding, and suffix expose stores outside the prediction block. + /// + /// The prediction geometry used to size the destination. + /// The padded destination-row stride. + /// The guarded destination storage. + private static byte[] CreateByteDestination(PredictionCase testCase, int destinationStride) + => Enumerable.Repeat(ByteDestinationSentinel, DestinationPrefix + (destinationStride * testCase.Height) + DestinationSuffix).ToArray(); + + /// + /// Creates a ushort destination whose prefix, row padding, and suffix expose stores outside the prediction block. + /// + /// The prediction geometry used to size the destination. + /// The padded destination-row stride. + /// The guarded destination storage. + private static ushort[] CreateHighBitDepthDestination(PredictionCase testCase, int destinationStride) + => Enumerable + .Repeat(HighBitDepthDestinationSentinel, DestinationPrefix + (destinationStride * testCase.Height) + DestinationSuffix) + .ToArray(); + + /// + /// Creates caller-owned two-dimensional intermediate storage using AV1's eight-tap vertical extent. + /// + /// The prediction geometry and fractional phases. + /// The required scratch storage, or an empty array for copy and one-dimensional predictions. + private static short[] CreateScratch(PredictionCase testCase) + => testCase.HorizontalPhase == 0 || testCase.VerticalPhase == 0 + ? [] + : new short[Math.Max(testCase.Width, 16) * (testCase.Height + FilterTapCount - 1)]; + + /// + /// Applies libaom's single-reference copy or convolution equations to an 8-bit prediction block. + /// + /// The complete padded reference plane. + /// The source-row stride. + /// The integer-position sample index. + /// The guarded destination storage. + /// The first active destination index. + /// The destination-row stride. + /// The prediction filters, phases, and geometry. + /// The decoded sample precision. + private static void ApplyReference( + byte[] source, + int sourceStride, + int sourceOrigin, + byte[] destination, + int destinationOrigin, + int destinationStride, + PredictionCase testCase, + int bitDepth) + { + if (testCase.HorizontalPhase == 0 && testCase.VerticalPhase == 0) + { + for (int row = 0; row < testCase.Height; row++) + { + source.AsSpan(sourceOrigin + (row * sourceStride), testCase.Width) + .CopyTo(destination.AsSpan(destinationOrigin + (row * destinationStride), testCase.Width)); + } + + return; + } + + if (testCase.VerticalPhase == 0) + { + ReadOnlySpan horizontal = GetCoefficients(testCase.HorizontalFilter, testCase.HorizontalPhase, testCase.Width <= 4); + int round0 = GetRound0Bits(bitDepth); + + for (int row = 0; row < testCase.Height; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sourceIndex = sourceOrigin + (row * sourceStride) + column - FilterCenterOffset; + int sum = Convolve(source, sourceIndex, 1, horizontal); + int value = RoundPowerOfTwo(RoundPowerOfTwo(sum, round0), FilterBits - round0); + destination[destinationOrigin + (row * destinationStride) + column] = (byte)Math.Clamp(value, 0, byte.MaxValue); + } + } + + return; + } + + if (testCase.HorizontalPhase == 0) + { + ReadOnlySpan vertical = GetCoefficients(testCase.VerticalFilter, testCase.VerticalPhase, testCase.Height <= 4); + + for (int row = 0; row < testCase.Height; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sourceIndex = sourceOrigin + ((row - FilterCenterOffset) * sourceStride) + column; + int sum = Convolve(source, sourceIndex, sourceStride, vertical); + int value = RoundPowerOfTwo(sum, FilterBits); + destination[destinationOrigin + (row * destinationStride) + column] = (byte)Math.Clamp(value, 0, byte.MaxValue); + } + } + + return; + } + + ReadOnlySpan horizontalCoefficients = GetCoefficients(testCase.HorizontalFilter, testCase.HorizontalPhase, testCase.Width <= 4); + ReadOnlySpan verticalCoefficients = GetCoefficients(testCase.VerticalFilter, testCase.VerticalPhase, testCase.Height <= 4); + + ApplyTwoDimensionalReference( + source, sourceStride, sourceOrigin, destination, destinationOrigin, destinationStride, testCase, + horizontalCoefficients, verticalCoefficients, bitDepth); + } + + /// + /// Applies libaom's single-reference copy or convolution equations to a high-bit-depth prediction block. + /// + /// The complete padded reference plane. + /// The source-row stride. + /// The integer-position sample index. + /// The guarded destination storage. + /// The first active destination index. + /// The destination-row stride. + /// The prediction filters, phases, and geometry. + /// The decoded sample precision. + private static void ApplyReference( + ushort[] source, + int sourceStride, + int sourceOrigin, + ushort[] destination, + int destinationOrigin, + int destinationStride, + PredictionCase testCase, + int bitDepth) + { + if (testCase.HorizontalPhase == 0 && testCase.VerticalPhase == 0) + { + for (int row = 0; row < testCase.Height; row++) + { + source.AsSpan(sourceOrigin + (row * sourceStride), testCase.Width) + .CopyTo(destination.AsSpan(destinationOrigin + (row * destinationStride), testCase.Width)); + } + + return; + } + + int maximum = (1 << bitDepth) - 1; + if (testCase.VerticalPhase == 0) + { + ReadOnlySpan horizontal = GetCoefficients(testCase.HorizontalFilter, testCase.HorizontalPhase, testCase.Width <= 4); + int round0 = GetRound0Bits(bitDepth); + + for (int row = 0; row < testCase.Height; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sourceIndex = sourceOrigin + (row * sourceStride) + column - FilterCenterOffset; + int sum = Convolve(source, sourceIndex, 1, horizontal); + int value = RoundPowerOfTwo(RoundPowerOfTwo(sum, round0), FilterBits - round0); + destination[destinationOrigin + (row * destinationStride) + column] = (ushort)Math.Clamp(value, 0, maximum); + } + } + + return; + } + + if (testCase.HorizontalPhase == 0) + { + ReadOnlySpan vertical = GetCoefficients(testCase.VerticalFilter, testCase.VerticalPhase, testCase.Height <= 4); + + for (int row = 0; row < testCase.Height; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sourceIndex = sourceOrigin + ((row - FilterCenterOffset) * sourceStride) + column; + int sum = Convolve(source, sourceIndex, sourceStride, vertical); + int value = RoundPowerOfTwo(sum, FilterBits); + destination[destinationOrigin + (row * destinationStride) + column] = (ushort)Math.Clamp(value, 0, maximum); + } + } + + return; + } + + ReadOnlySpan horizontalCoefficients = GetCoefficients(testCase.HorizontalFilter, testCase.HorizontalPhase, testCase.Width <= 4); + ReadOnlySpan verticalCoefficients = GetCoefficients(testCase.VerticalFilter, testCase.VerticalPhase, testCase.Height <= 4); + + ApplyTwoDimensionalReference( + source, sourceStride, sourceOrigin, destination, destinationOrigin, destinationStride, testCase, + horizontalCoefficients, verticalCoefficients, bitDepth); + } + + /// + /// Applies libaom's biased two-pass 8-bit convolution and removes both intermediate bias terms after vertical filtering. + /// + /// The complete padded reference plane. + /// The source-row stride. + /// The integer-position sample index. + /// The guarded destination storage. + /// The first active destination index. + /// The destination-row stride. + /// The prediction geometry and phases. + /// The horizontal Q7 coefficient row. + /// The vertical Q7 coefficient row. + /// The decoded sample precision. + private static void ApplyTwoDimensionalReference( + byte[] source, + int sourceStride, + int sourceOrigin, + byte[] destination, + int destinationOrigin, + int destinationStride, + PredictionCase testCase, + ReadOnlySpan horizontalCoefficients, + ReadOnlySpan verticalCoefficients, + int bitDepth) + { + short[] intermediate = new short[(testCase.Height + FilterTapCount - 1) * testCase.Width]; + int horizontalBias = 1 << (bitDepth + FilterBits - 1); + int round0 = GetRound0Bits(bitDepth); + + for (int row = 0; row < testCase.Height + FilterTapCount - 1; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sourceIndex = sourceOrigin + ((row - FilterCenterOffset) * sourceStride) + column - FilterCenterOffset; + int sum = horizontalBias + Convolve(source, sourceIndex, 1, horizontalCoefficients); + intermediate[(row * testCase.Width) + column] = (short)RoundPowerOfTwo(sum, round0); + } + } + + WriteTwoDimensionalReference(intermediate, destination, destinationOrigin, destinationStride, testCase, verticalCoefficients, bitDepth); + } + + /// + /// Applies libaom's biased two-pass high-bit-depth convolution and removes both intermediate bias terms after vertical filtering. + /// + /// The complete padded reference plane. + /// The source-row stride. + /// The integer-position sample index. + /// The guarded destination storage. + /// The first active destination index. + /// The destination-row stride. + /// The prediction geometry and phases. + /// The horizontal Q7 coefficient row. + /// The vertical Q7 coefficient row. + /// The decoded sample precision. + private static void ApplyTwoDimensionalReference( + ushort[] source, + int sourceStride, + int sourceOrigin, + ushort[] destination, + int destinationOrigin, + int destinationStride, + PredictionCase testCase, + ReadOnlySpan horizontalCoefficients, + ReadOnlySpan verticalCoefficients, + int bitDepth) + { + short[] intermediate = new short[(testCase.Height + FilterTapCount - 1) * testCase.Width]; + int horizontalBias = 1 << (bitDepth + FilterBits - 1); + int round0 = GetRound0Bits(bitDepth); + + for (int row = 0; row < testCase.Height + FilterTapCount - 1; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sourceIndex = sourceOrigin + ((row - FilterCenterOffset) * sourceStride) + column - FilterCenterOffset; + int sum = horizontalBias + Convolve(source, sourceIndex, 1, horizontalCoefficients); + intermediate[(row * testCase.Width) + column] = (short)RoundPowerOfTwo(sum, round0); + } + } + + WriteTwoDimensionalReference(intermediate, destination, destinationOrigin, destinationStride, testCase, verticalCoefficients, bitDepth); + } + + /// + /// Completes an 8-bit two-dimensional prediction from the independently generated biased intermediate block. + /// + /// The horizontally filtered signed intermediate block. + /// The guarded destination storage. + /// The first active destination index. + /// The destination-row stride. + /// The prediction geometry. + /// The vertical Q7 coefficient row. + /// The decoded sample precision. + private static void WriteTwoDimensionalReference( + short[] intermediate, + byte[] destination, + int destinationOrigin, + int destinationStride, + PredictionCase testCase, + ReadOnlySpan verticalCoefficients, + int bitDepth) + { + int round0 = GetRound0Bits(bitDepth); + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + int verticalBias = 1 << offsetBits; + int roundOffset = (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1)); + + for (int row = 0; row < testCase.Height; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sum = verticalBias + Convolve(intermediate, (row * testCase.Width) + column, testCase.Width, verticalCoefficients); + int value = RoundPowerOfTwo(sum, round1) - roundOffset; + destination[destinationOrigin + (row * destinationStride) + column] = (byte)Math.Clamp(value, 0, byte.MaxValue); + } + } + } + + /// + /// Completes a high-bit-depth two-dimensional prediction from the independently generated biased intermediate block. + /// + /// The horizontally filtered signed intermediate block. + /// The guarded destination storage. + /// The first active destination index. + /// The destination-row stride. + /// The prediction geometry. + /// The vertical Q7 coefficient row. + /// The decoded sample precision. + private static void WriteTwoDimensionalReference( + short[] intermediate, + ushort[] destination, + int destinationOrigin, + int destinationStride, + PredictionCase testCase, + ReadOnlySpan verticalCoefficients, + int bitDepth) + { + int round0 = GetRound0Bits(bitDepth); + int round1 = (2 * FilterBits) - round0; + int offsetBits = bitDepth + (2 * FilterBits) - round0; + int verticalBias = 1 << offsetBits; + int roundOffset = (1 << (offsetBits - round1)) + (1 << (offsetBits - round1 - 1)); + int maximum = (1 << bitDepth) - 1; + + for (int row = 0; row < testCase.Height; row++) + { + for (int column = 0; column < testCase.Width; column++) + { + int sum = verticalBias + Convolve(intermediate, (row * testCase.Width) + column, testCase.Width, verticalCoefficients); + int value = RoundPowerOfTwo(sum, round1) - roundOffset; + destination[destinationOrigin + (row * destinationStride) + column] = (ushort)Math.Clamp(value, 0, maximum); + } + } + } + + /// + /// Computes one eight-tap Q7 convolution from 8-bit samples. + /// + /// The complete source storage. + /// The first coefficient's source index. + /// The source-element distance between taps. + /// The eight Q7 coefficients. + /// The unrounded convolution sum. + private static int Convolve(byte[] source, int sourceIndex, int sourceStep, ReadOnlySpan coefficients) + { + int sum = 0; + for (int tap = 0; tap < FilterTapCount; tap++) + { + sum += source[sourceIndex + (tap * sourceStep)] * coefficients[tap]; + } + + return sum; + } + + /// + /// Computes one eight-tap Q7 convolution from high-bit-depth samples. + /// + /// The complete source storage. + /// The first coefficient's source index. + /// The source-element distance between taps. + /// The eight Q7 coefficients. + /// The unrounded convolution sum. + private static int Convolve(ushort[] source, int sourceIndex, int sourceStep, ReadOnlySpan coefficients) + { + int sum = 0; + for (int tap = 0; tap < FilterTapCount; tap++) + { + sum += source[sourceIndex + (tap * sourceStep)] * coefficients[tap]; + } + + return sum; + } + + /// + /// Computes one eight-tap Q7 convolution from signed biased intermediate samples. + /// + /// The complete intermediate storage. + /// The first coefficient's source index. + /// The source-element distance between taps. + /// The eight Q7 coefficients. + /// The unrounded convolution sum. + private static int Convolve(short[] source, int sourceIndex, int sourceStep, ReadOnlySpan coefficients) + { + int sum = 0; + for (int tap = 0; tap < FilterTapCount; tap++) + { + sum += source[sourceIndex + (tap * sourceStep)] * coefficients[tap]; + } + + return sum; + } + + /// + /// Selects one normative Q7 coefficient row independently of the production filter storage. + /// + /// The interpolation-filter family. + /// The one-sixteenth-sample phase. + /// A value indicating whether the dimension is four samples. + /// The eight-position Q7 coefficient row. + private static ReadOnlySpan GetCoefficients(Av1InterpolationFilter filter, int phase, bool useReducedFilter) + { + if (filter == Av1InterpolationFilter.Bilinear) + { + return phase switch + { + 3 => BilinearPhase3, + 8 => BilinearPhase8, + 11 => BilinearPhase11, + 15 => BilinearPhase15, + _ => throw new InvalidOperationException($"The test oracle has no bilinear coefficient row for phase {phase}.") + }; + } + + if (useReducedFilter && filter == Av1InterpolationFilter.Sharp) + { + // AV1 maps sharp filtering on a four-sample dimension to the reduced regular table before convolution. + filter = Av1InterpolationFilter.Regular; + } + + return (filter, useReducedFilter, phase) switch + { + (Av1InterpolationFilter.Regular, false, 1) => RegularEightTapPhase1, + (Av1InterpolationFilter.Smooth, false, 7) => SmoothEightTapPhase7, + (Av1InterpolationFilter.Smooth, false, 15) => SmoothEightTapPhase15, + (Av1InterpolationFilter.Sharp, false, 8) => SharpEightTapPhase8, + (Av1InterpolationFilter.Regular, true, 3) => RegularFourTapPhase3, + (Av1InterpolationFilter.Regular, true, 5) => RegularFourTapPhase5, + (Av1InterpolationFilter.Regular, true, 13) => RegularFourTapPhase13, + (Av1InterpolationFilter.Smooth, true, 7) => SmoothFourTapPhase7, + (Av1InterpolationFilter.Smooth, true, 13) => SmoothFourTapPhase13, + _ => throw new InvalidOperationException( + $"The test oracle has no coefficient row for {filter}, phase {phase}, reduced {useReducedFilter}.") + }; + } + + /// + /// Gets the regular eight-tap Q7 kernel for phase 1 from AOM's normative decoder table. + /// + private static ReadOnlySpan RegularEightTapPhase1 => [0, 2, -6, 126, 8, -2, 0, 0]; + + /// + /// Gets the smooth eight-tap Q7 kernel for phase 7 from AOM's normative decoder table. + /// + private static ReadOnlySpan SmoothEightTapPhase7 => [0, -2, 16, 54, 48, 12, 0, 0]; + + /// + /// Gets the smooth eight-tap Q7 kernel for phase 15 from AOM's normative decoder table. + /// + private static ReadOnlySpan SmoothEightTapPhase15 => [0, 0, 2, 34, 62, 28, 2, 0]; + + /// + /// Gets the sharp eight-tap Q7 kernel for phase 8 from AOM's normative decoder table. + /// + private static ReadOnlySpan SharpEightTapPhase8 => [-4, 12, -24, 80, 80, -24, 12, -4]; + + /// + /// Gets the bilinear Q7 kernel for phase 3 from AOM's normative decoder table. + /// + private static ReadOnlySpan BilinearPhase3 => [0, 0, 0, 104, 24, 0, 0, 0]; + + /// + /// Gets the bilinear Q7 kernel for phase 8 from AOM's normative decoder table. + /// + private static ReadOnlySpan BilinearPhase8 => [0, 0, 0, 64, 64, 0, 0, 0]; + + /// + /// Gets the bilinear Q7 kernel for phase 11 from AOM's normative decoder table. + /// + private static ReadOnlySpan BilinearPhase11 => [0, 0, 0, 40, 88, 0, 0, 0]; + + /// + /// Gets the bilinear Q7 kernel for phase 15 from AOM's normative decoder table. + /// + private static ReadOnlySpan BilinearPhase15 => [0, 0, 0, 8, 120, 0, 0, 0]; + + /// + /// Gets the reduced regular Q7 kernel for phase 3 from AOM's normative decoder table. + /// + private static ReadOnlySpan RegularFourTapPhase3 => [0, 0, -10, 116, 28, -6, 0, 0]; + + /// + /// Gets the reduced regular Q7 kernel for phase 5 from AOM's normative decoder table. + /// + private static ReadOnlySpan RegularFourTapPhase5 => [0, 0, -12, 102, 48, -10, 0, 0]; + + /// + /// Gets the reduced regular Q7 kernel for phase 13 from AOM's normative decoder table. + /// + private static ReadOnlySpan RegularFourTapPhase13 => [0, 0, -6, 28, 116, -10, 0, 0]; + + /// + /// Gets the reduced smooth Q7 kernel for phase 7 from AOM's normative decoder table. + /// + private static ReadOnlySpan SmoothFourTapPhase7 => [0, 0, 14, 54, 48, 12, 0, 0]; + + /// + /// Gets the reduced smooth Q7 kernel for phase 13 from AOM's normative decoder table. + /// + private static ReadOnlySpan SmoothFourTapPhase13 => [0, 0, 4, 40, 62, 22, 0, 0]; + + /// + /// Gets AOM's first convolution shift while keeping the biased intermediate within sixteen signed bits. + /// + /// The decoded sample precision. + /// The first convolution shift. + private static int GetRound0Bits(int bitDepth) + { + int round0 = Round0Bits; + int intermediateBitCount = bitDepth + FilterBits - round0 + 2; + if (intermediateBitCount > 16) + { + round0 += intermediateBitCount - 16; + } + + return round0; + } + + /// + /// Applies AOM's integer power-of-two rounding rule. + /// + /// The signed integer to divide. + /// The base-2 divisor exponent. + /// The rounded quotient. + private static int RoundPowerOfTwo(int value, int bits) => (value + (1 << (bits - 1))) >> bits; + + /// + /// Reports the first differing byte, including guarded padding, for one named prediction path. + /// + /// The independently generated destination storage. + /// The production destination storage. + /// The prediction scenario. + /// The production execution path. + private static void AssertEqual(byte[] expected, byte[] actual, PredictionCase testCase, string path) + { + for (int i = 0; i < expected.Length; i++) + { + if (expected[i] != actual[i]) + { + Assert.Fail($"{path} prediction '{testCase.Name}' differs at storage index {i}: expected {expected[i]}, actual {actual[i]}."); + } + } + } + + /// + /// Reports the first differing ushort, including guarded padding, for one named prediction path. + /// + /// The independently generated destination storage. + /// The production destination storage. + /// The prediction scenario. + /// The production execution path. + private static void AssertEqual(ushort[] expected, ushort[] actual, PredictionCase testCase, string path) + { + for (int i = 0; i < expected.Length; i++) + { + if (expected[i] != actual[i]) + { + Assert.Fail($"{path} prediction '{testCase.Name}' differs at storage index {i}: expected {expected[i]}, actual {actual[i]}."); + } + } + } + + /// + /// Describes one prediction path, filter pair, phase pair, and block geometry. + /// + private readonly struct PredictionCase + { + /// + /// Initializes a new instance of the struct. + /// + /// The diagnostic scenario name. + /// The active prediction width. + /// The active prediction height. + /// The horizontal interpolation-filter family. + /// The vertical interpolation-filter family. + /// The horizontal one-sixteenth-sample phase. + /// The vertical one-sixteenth-sample phase. + public PredictionCase( + string name, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase) + { + this.Name = name; + this.Width = width; + this.Height = height; + this.HorizontalFilter = horizontalFilter; + this.VerticalFilter = verticalFilter; + this.HorizontalPhase = horizontalPhase; + this.VerticalPhase = verticalPhase; + } + + /// + /// Gets the diagnostic scenario name. + /// + public string Name { get; } + + /// + /// Gets the active prediction width. + /// + public int Width { get; } + + /// + /// Gets the active prediction height. + /// + public int Height { get; } + + /// + /// Gets the horizontal interpolation-filter family. + /// + public Av1InterpolationFilter HorizontalFilter { get; } + + /// + /// Gets the vertical interpolation-filter family. + /// + public Av1InterpolationFilter VerticalFilter { get; } + + /// + /// Gets the horizontal one-sixteenth-sample phase. + /// + public int HorizontalPhase { get; } + + /// + /// Gets the vertical one-sixteenth-sample phase. + /// + public int VerticalPhase { get; } + } +}