diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 7a5b2ed474..a919951b59 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -827,7 +827,8 @@ Encoder verification contract: - [~] Tile partition writing now follows current libaom's recursive `write_modes_sb` preorder traversal and `update_ext_partition_context` edge updates directly. The obsolete SVT-derived global geometry catalog and its unimplemented lookup are removed; transform geometry is derived in libaom's bounded 64x64 residual order, fixed intra transform-size symbols use the reference depth and neighbor contexts, frame-edge and segmentation syntax use mode-information units, and 128x128 CDEF units use libaom's 0-to-3 indexing and first-block strength ownership. Partition and mode analysis still need to populate these retained decisions; variable inter-transform syntax remains part of later inter-frame support. - [ ] Implement legal deblocking, CDEF, restoration, super-resolution, and film-grain signaling decisions. - [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform, defers both coefficient rents until the first nonzero transform block, and disposes all tile scratch independently from the detached encoded bytes. Its range coder matches current libaom's 64-bit coding window, bulk big-endian byte flush, and backward carry propagation while using one byte of allocator scratch per estimated output byte instead of the former 16-bit pre-carry storage. The reference-type symbol encoder is passed normally through tile traversal, and the operation boundary owns the allocator-backed item payload stream for exactly one synchronous encode. Every remaining encoder fragment must be audited before it becomes active. -- [~] The planar conversion, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Apply the same rule to every later hot-path family. +- [~] The planar conversion, residual construction, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Residual construction matches current libaom's exact source-minus-prediction arithmetic for 8-bit and high-bit-depth planes, preserves independent row strides and unaligned starts, and writes directly into caller-owned signed-short storage without allocation. Apply the same rule to every later hot-path family. +- [~] Residual tests verify misaligned planes, independent source, prediction, and destination strides, SIMD remainders, untouched padding, 8-bit, 10-bit, and 12-bit precision, every operator width independently of host acceleration, the scalar fallback, and zero per-transform allocations. - [~] Forward-quantizer FeatureTestRunner and zero-allocation tests compare every hardware tier with an independent scan-order scalar oracle shaped from current-main libaom. Both passed direct net11 VSTest in Release. - [~] The combined-frame writer now completes the byte-counted uncompressed frame header before starting the optional multi-tile tile-group flag, matching current libaom's separate frame-header and tile-group writers. A non-uniform two-tile round trip verifies the explicit boundaries, both tile payloads, and complete stream consumption through direct net11 VSTest in Release. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs new file mode 100644 index 0000000000..4e512abba9 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs @@ -0,0 +1,118 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Defines the sample-width-specific arithmetic used by . +/// +internal static partial class Av1ResidualBuilder +{ + /// + /// Defines one AV1 source-minus-prediction operation across hardware widths. + /// + /// The source and prediction sample type. + internal interface IResidualOperator + where TSample : unmanaged + { + /// + /// Subtracts eight or sixteen source and prediction samples. + /// + /// The source samples. + /// The prediction samples. + /// The upper residual lanes when the inputs contain 8-bit samples. + /// The lower residual lanes. + public static abstract Vector128 Subtract(Vector128 source, Vector128 prediction, out Vector128 upper); + + /// + /// Subtracts sixteen or thirty-two source and prediction samples. + /// + /// The source samples. + /// The prediction samples. + /// The upper residual lanes when the inputs contain 8-bit samples. + /// The lower residual lanes. + public static abstract Vector256 Subtract(Vector256 source, Vector256 prediction, out Vector256 upper); + + /// + /// Subtracts thirty-two or sixty-four source and prediction samples. + /// + /// The source samples. + /// The prediction samples. + /// The upper residual lanes when the inputs contain 8-bit samples. + /// The lower residual lanes. + public static abstract Vector512 Subtract(Vector512 source, Vector512 prediction, out Vector512 upper); + + /// + /// Subtracts one source and prediction sample. + /// + /// The source sample. + /// The prediction sample. + /// The signed residual. + public static abstract short Subtract(TSample source, TSample prediction); + } + + /// + /// Widens 8-bit samples before subtraction so every residual is represented without precision loss. + /// + internal readonly struct ByteOperator : IResidualOperator + { + /// + public static Vector128 Subtract(Vector128 source, Vector128 prediction, out Vector128 upper) + { + Vector128 lower = Vector128.WidenLower(source).AsInt16() - Vector128.WidenLower(prediction).AsInt16(); + upper = Vector128.WidenUpper(source).AsInt16() - Vector128.WidenUpper(prediction).AsInt16(); + return lower; + } + + /// + public static Vector256 Subtract(Vector256 source, Vector256 prediction, out Vector256 upper) + { + Vector256 lower = Vector256.WidenLower(source).AsInt16() - Vector256.WidenLower(prediction).AsInt16(); + upper = Vector256.WidenUpper(source).AsInt16() - Vector256.WidenUpper(prediction).AsInt16(); + return lower; + } + + /// + public static Vector512 Subtract(Vector512 source, Vector512 prediction, out Vector512 upper) + { + Vector512 lower = Vector512.WidenLower(source).AsInt16() - Vector512.WidenLower(prediction).AsInt16(); + upper = Vector512.WidenUpper(source).AsInt16() - Vector512.WidenUpper(prediction).AsInt16(); + return lower; + } + + /// + public static short Subtract(byte source, byte prediction) => (short)(source - prediction); + } + + /// + /// Subtracts high-bit-depth samples directly because AV1's 10-bit and 12-bit ranges fit signed-short lanes. + /// + internal readonly struct UInt16Operator : IResidualOperator + { + /// + public static Vector128 Subtract(Vector128 source, Vector128 prediction, out Vector128 upper) + { + upper = default; + return source.AsInt16() - prediction.AsInt16(); + } + + /// + public static Vector256 Subtract(Vector256 source, Vector256 prediction, out Vector256 upper) + { + upper = default; + return source.AsInt16() - prediction.AsInt16(); + } + + /// + public static Vector512 Subtract(Vector512 source, Vector512 prediction, out Vector512 upper) + { + upper = default; + return source.AsInt16() - prediction.AsInt16(); + } + + /// + public static short Subtract(ushort source, ushort prediction) => (short)(source - prediction); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs new file mode 100644 index 0000000000..c22343add1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs @@ -0,0 +1,150 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Builds signed AV1 residual planes from source and prediction samples. +/// +internal static partial class Av1ResidualBuilder +{ + /// + /// Subtracts an 8-bit prediction plane from its source plane. + /// + /// The source samples. + /// The source row stride. + /// The prediction samples. + /// The prediction row stride. + /// The destination residual samples. + /// The residual row stride. + /// The number of samples per row. + /// The number of rows. + public static void Subtract( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + Span residual, + int residualStride, + int width, + int height) + => Subtract(source, sourceStride, prediction, predictionStride, residual, residualStride, width, height); + + /// + /// Subtracts a high-bit-depth prediction plane from its source plane. + /// + /// The source samples. + /// The source row stride. + /// The prediction samples. + /// The prediction row stride. + /// The destination residual samples. + /// The residual row stride. + /// The number of samples per row. + /// The number of rows. + public static void Subtract( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + Span residual, + int residualStride, + int width, + int height) + => Subtract(source, sourceStride, prediction, predictionStride, residual, residualStride, width, height); + + private static void Subtract( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + Span residual, + int residualStride, + int width, + int height) + where TSample : unmanaged + where TOperator : struct, IResidualOperator + { + for (int y = 0; y < height; y++) + { + ReadOnlySpan sourceRow = source.Slice(y * sourceStride, width); + ReadOnlySpan predictionRow = prediction.Slice(y * predictionStride, width); + Span residualRow = residual.Slice(y * residualStride, width); + + ref TSample sourceBase = ref MemoryMarshal.GetReference(sourceRow); + ref TSample predictionBase = ref MemoryMarshal.GetReference(predictionRow); + ref short residualBase = ref MemoryMarshal.GetReference(residualRow); + int x = 0; + + // Each narrower tier resumes at the shared sample offset, preserving SIMD execution for the widest + // possible remainder while leaving only a sub-vector tail for scalar subtraction. + if (Vector512.IsHardwareAccelerated) + { + nuint vectorCount = sourceRow.Vector512Count(); + + for (; vectorCount > 0; vectorCount--, x += Vector512.Count) + { + Vector512 sourceVector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, x)); + Vector512 predictionVector = Unsafe.As>(ref Unsafe.Add(ref predictionBase, x)); + Vector512 lower = TOperator.Subtract(sourceVector, predictionVector, out Vector512 upper); + + Unsafe.As>(ref Unsafe.Add(ref residualBase, x)) = lower; + + // Byte vectors widen into two signed-short vectors; high-bit-depth vectors retain one lane per sample. + if (Vector512.Count != Vector512.Count) + { + Unsafe.As>(ref Unsafe.Add(ref residualBase, x + Vector512.Count)) = upper; + } + } + } + + if (Vector256.IsHardwareAccelerated) + { + nuint vectorCount = sourceRow[x..].Vector256Count(); + + for (; vectorCount > 0; vectorCount--, x += Vector256.Count) + { + Vector256 sourceVector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, x)); + Vector256 predictionVector = Unsafe.As>(ref Unsafe.Add(ref predictionBase, x)); + Vector256 lower = TOperator.Subtract(sourceVector, predictionVector, out Vector256 upper); + + Unsafe.As>(ref Unsafe.Add(ref residualBase, x)) = lower; + + if (Vector256.Count != Vector256.Count) + { + Unsafe.As>(ref Unsafe.Add(ref residualBase, x + Vector256.Count)) = upper; + } + } + } + + if (Vector128.IsHardwareAccelerated) + { + nuint vectorCount = sourceRow[x..].Vector128Count(); + + for (; vectorCount > 0; vectorCount--, x += Vector128.Count) + { + Vector128 sourceVector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, x)); + Vector128 predictionVector = Unsafe.As>(ref Unsafe.Add(ref predictionBase, x)); + Vector128 lower = TOperator.Subtract(sourceVector, predictionVector, out Vector128 upper); + + Unsafe.As>(ref Unsafe.Add(ref residualBase, x)) = lower; + + if (Vector128.Count != Vector128.Count) + { + Unsafe.As>(ref Unsafe.Add(ref residualBase, x + Vector128.Count)) = upper; + } + } + } + + for (; x < width; x++) + { + Unsafe.Add(ref residualBase, x) = TOperator.Subtract( + Unsafe.Add(ref sourceBase, x), + Unsafe.Add(ref predictionBase, x)); + } + } + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs new file mode 100644 index 0000000000..14b756c7d1 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs @@ -0,0 +1,275 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; +using SixLabors.ImageSharp.Tests.TestUtilities; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies AV1 residual construction against source-minus-prediction reference arithmetic. +/// +[Trait("Format", "Avif")] +public class Av1ResidualBuilderTests +{ + private const HwIntrinsics ResidualConfigurations = + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + + /// + /// Verifies 8-bit, 10-bit, and 12-bit residuals across misaligned planes, independent strides, and SIMD tails. + /// + [Fact] + public void ResidualsMatchReferenceAcrossHardwareWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateResiduals, ResidualConfigurations); + + /// + /// Verifies every width-specific operator even when the current processor cannot select that width in the driver. + /// + [Fact] + public void ResidualOperatorsMatchReferenceAtEveryVectorWidth() + { + byte[] byteSource = new byte[Vector512.Count]; + byte[] bytePrediction = new byte[Vector512.Count]; + short[] byteExpected = new short[Vector512.Count]; + short[] byteActual = new short[Vector512.Count]; + FillBytePlanes(byteSource, byteSource.Length, bytePrediction, bytePrediction.Length, byteSource.Length, 1); + FillReference(byteSource, byteSource.Length, bytePrediction, bytePrediction.Length, byteExpected, byteExpected.Length, byteExpected.Length, 1); + + ref byte byteSourceBase = ref MemoryMarshal.GetArrayDataReference(byteSource); + ref byte bytePredictionBase = ref MemoryMarshal.GetArrayDataReference(bytePrediction); + Vector128 byteLower128 = Av1ResidualBuilder.ByteOperator.Subtract( + Unsafe.As>(ref byteSourceBase), + Unsafe.As>(ref bytePredictionBase), + out Vector128 byteUpper128); + + byteLower128.CopyTo(byteActual); + byteUpper128.CopyTo(byteActual.AsSpan(Vector128.Count)); + AssertEqual(byteExpected, byteActual, Vector128.Count); + + Vector256 byteLower256 = Av1ResidualBuilder.ByteOperator.Subtract( + Unsafe.As>(ref byteSourceBase), + Unsafe.As>(ref bytePredictionBase), + out Vector256 byteUpper256); + + byteLower256.CopyTo(byteActual); + byteUpper256.CopyTo(byteActual.AsSpan(Vector256.Count)); + AssertEqual(byteExpected, byteActual, Vector256.Count); + + Vector512 byteLower512 = Av1ResidualBuilder.ByteOperator.Subtract( + Unsafe.As>(ref byteSourceBase), + Unsafe.As>(ref bytePredictionBase), + out Vector512 byteUpper512); + + byteLower512.CopyTo(byteActual); + byteUpper512.CopyTo(byteActual.AsSpan(Vector512.Count)); + AssertEqual(byteExpected, byteActual, Vector512.Count); + + ushort[] uint16Source = new ushort[Vector512.Count]; + ushort[] uint16Prediction = new ushort[Vector512.Count]; + short[] uint16Expected = new short[Vector512.Count]; + short[] uint16Actual = new short[Vector512.Count]; + FillUInt16Planes(uint16Source, uint16Source.Length, uint16Prediction, uint16Prediction.Length, uint16Source.Length, 1, 4095); + FillReference(uint16Source, uint16Source.Length, uint16Prediction, uint16Prediction.Length, uint16Expected, uint16Expected.Length, uint16Expected.Length, 1); + + ref ushort uint16SourceBase = ref MemoryMarshal.GetArrayDataReference(uint16Source); + ref ushort uint16PredictionBase = ref MemoryMarshal.GetArrayDataReference(uint16Prediction); + Av1ResidualBuilder.UInt16Operator.Subtract( + Unsafe.As>(ref uint16SourceBase), + Unsafe.As>(ref uint16PredictionBase), + out _).CopyTo(uint16Actual); + + AssertEqual(uint16Expected, uint16Actual, Vector128.Count); + + Av1ResidualBuilder.UInt16Operator.Subtract( + Unsafe.As>(ref uint16SourceBase), + Unsafe.As>(ref uint16PredictionBase), + out _).CopyTo(uint16Actual); + + AssertEqual(uint16Expected, uint16Actual, Vector256.Count); + + Av1ResidualBuilder.UInt16Operator.Subtract( + Unsafe.As>(ref uint16SourceBase), + Unsafe.As>(ref uint16PredictionBase), + out _).CopyTo(uint16Actual); + + AssertEqual(uint16Expected, uint16Actual, Vector512.Count); + } + + /// + /// Verifies that repeated maximum-transform residual construction uses only caller-owned buffers. + /// + [Fact] + public void ResidualConstructionDoesNotAllocate() + { + const int width = 64; + const int height = 64; + byte[] source = new byte[width * height]; + byte[] prediction = new byte[width * height]; + short[] residual = new short[width * height]; + ushort[] highBitDepthSource = new ushort[width * height]; + ushort[] highBitDepthPrediction = new ushort[width * height]; + short[] highBitDepthResidual = new short[width * height]; + FillBytePlanes(source, width, prediction, width, width, height); + FillUInt16Planes(highBitDepthSource, width, highBitDepthPrediction, width, width, height, 4095); + + Av1ResidualBuilder.Subtract(source, width, prediction, width, residual, width, width, height); + Av1ResidualBuilder.Subtract(highBitDepthSource, width, highBitDepthPrediction, width, highBitDepthResidual, width, width, height); + + long before = GC.GetAllocatedBytesForCurrentThread(); + for (int iteration = 0; iteration < 32; iteration++) + { + Av1ResidualBuilder.Subtract(source, width, prediction, width, residual, width, width, height); + Av1ResidualBuilder.Subtract(highBitDepthSource, width, highBitDepthPrediction, width, highBitDepthResidual, width, width, height); + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + } + + private static void ValidateResiduals() + { + ValidateByteResiduals(); + ValidateUInt16Residuals(1023); + ValidateUInt16Residuals(4095); + } + + private static void ValidateByteResiduals() + { + const int width = 127; + const int height = 3; + const int sourceStride = 131; + const int predictionStride = 137; + const int residualStride = 139; + const int sourceOffset = 1; + const int predictionOffset = 2; + const int residualOffset = 3; + byte[] source = new byte[sourceOffset + (sourceStride * height)]; + byte[] prediction = new byte[predictionOffset + (predictionStride * height)]; + short[] expected = new short[residualOffset + (residualStride * height)]; + short[] actual = new short[expected.Length]; + Array.Fill(expected, short.MinValue); + Array.Fill(actual, short.MinValue); + + Span sourcePlane = source.AsSpan(sourceOffset); + Span predictionPlane = prediction.AsSpan(predictionOffset); + Span expectedPlane = expected.AsSpan(residualOffset); + Span actualPlane = actual.AsSpan(residualOffset); + FillBytePlanes(sourcePlane, sourceStride, predictionPlane, predictionStride, width, height); + FillReference(sourcePlane, sourceStride, predictionPlane, predictionStride, expectedPlane, residualStride, width, height); + + Av1ResidualBuilder.Subtract(sourcePlane, sourceStride, predictionPlane, predictionStride, actualPlane, residualStride, width, height); + + Assert.Equal(expected, actual); + } + + private static void ValidateUInt16Residuals(int maximumSample) + { + const int width = 127; + const int height = 3; + const int sourceStride = 131; + const int predictionStride = 137; + const int residualStride = 139; + const int sourceOffset = 1; + const int predictionOffset = 2; + const int residualOffset = 3; + ushort[] source = new ushort[sourceOffset + (sourceStride * height)]; + ushort[] prediction = new ushort[predictionOffset + (predictionStride * height)]; + short[] expected = new short[residualOffset + (residualStride * height)]; + short[] actual = new short[expected.Length]; + Array.Fill(expected, short.MinValue); + Array.Fill(actual, short.MinValue); + + Span sourcePlane = source.AsSpan(sourceOffset); + Span predictionPlane = prediction.AsSpan(predictionOffset); + Span expectedPlane = expected.AsSpan(residualOffset); + Span actualPlane = actual.AsSpan(residualOffset); + FillUInt16Planes(sourcePlane, sourceStride, predictionPlane, predictionStride, width, height, maximumSample); + FillReference(sourcePlane, sourceStride, predictionPlane, predictionStride, expectedPlane, residualStride, width, height); + + Av1ResidualBuilder.Subtract(sourcePlane, sourceStride, predictionPlane, predictionStride, actualPlane, residualStride, width, height); + + Assert.Equal(expected, actual); + } + + private static void FillBytePlanes(Span source, int sourceStride, Span prediction, int predictionStride, int width, int height) + { + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + source[(y * sourceStride) + x] = (byte)(((x * 37) + (y * 19) + 251) & byte.MaxValue); + prediction[(y * predictionStride) + x] = (byte)(((x * 11) + (y * 43) + 127) & byte.MaxValue); + } + } + + source[0] = byte.MaxValue; + prediction[0] = 0; + source[1] = 0; + prediction[1] = byte.MaxValue; + } + + private static void FillUInt16Planes(Span source, int sourceStride, Span prediction, int predictionStride, int width, int height, int maximumSample) + { + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + source[(y * sourceStride) + x] = (ushort)(((x * 197) + (y * 389) + maximumSample) & maximumSample); + prediction[(y * predictionStride) + x] = (ushort)(((x * 283) + (y * 151) + (maximumSample / 2)) & maximumSample); + } + } + + source[0] = (ushort)maximumSample; + prediction[0] = 0; + source[1] = 0; + prediction[1] = (ushort)maximumSample; + } + + private static void FillReference( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + Span residual, + int residualStride, + int width, + int height) + { + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + residual[(y * residualStride) + x] = (short)(source[(y * sourceStride) + x] - prediction[(y * predictionStride) + x]); + } + } + } + + private static void FillReference( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + Span residual, + int residualStride, + int width, + int height) + { + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + residual[(y * residualStride) + x] = (short)(source[(y * sourceStride) + x] - prediction[(y * predictionStride) + x]); + } + } + } + + private static void AssertEqual(ReadOnlySpan expected, ReadOnlySpan actual, int count) + { + for (int i = 0; i < count; i++) + { + Assert.Equal(expected[i], actual[i]); + } + } +}