diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 41fa72646b..aad4d0550d 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -874,6 +874,8 @@ Encoder verification contract: - [x] Uniform four-by-four luma transform search now uses two compact reconstruction and coefficient views already available in the aligned mode-decision workspace. Legal transform trials write into the non-winning view and exchange span ownership only on strict rate-distortion improvement; the selected coefficients and strided reconstruction mosaic are published once after the type search so the next raster transform sees the required decoded edge. This removes reconstruction and coefficient copies on every improving transform without adding storage, changing tie order, or repeating a transform. All four representative palette, filter-intra, and effort-eight output hashes are unchanged, the focused Release set passes 13 of 13 cases, and current-main `aomdec` accepts every checked stream. +- [x] Candidate distortion now follows the reference separation between immutable source residuals and reconstructed-pixel error. The forward-transform boundary accepts a read-only residual, so all transform types for one prediction reuse that block directly instead of copying it into transform scratch before every trial. The shared residual API now measures strided source-versus-reconstruction squared error with documented Vector512, Vector256, Vector128, and scalar traversal through ImageSharp's vector-count helpers, eliminating the former residual destination write and second reduction pass. The same path covers ordinary intra, filter intra, palette, chroma-from-luma, split transforms, and intra-block copy at 8, 10, and 12 bits without an allocation. Independent scalar, stride, tail, intrinsic-tier, and zero-allocation coverage passes with the 140-case focused encoder set; the complete non-HEVC HEIF/AV1 namespace passes 9,301 of 9,301. Representative palette, filter-intra, and effort-eight output hashes remain byte-identical, and current-main `aomdec` accepts every checked stream. + ### 7. Write complete AVIF output - [~] The encoder-side AV1 codec configuration is now derived directly from the encoded sequence header and writes the fixed four-byte `av1C` record with empty `configOBUs`. The image payload retains the required sequence header, so the property introduces no sequence-header allocation, retention, or copy. Four production-header cases cover main, high, and professional profiles; 8-, 10-, and 12-bit precision; monochrome, 4:2:0, 4:2:2, and 4:4:4 sampling; exact fixed bytes; decoder reparsing; and header/property equivalence through direct net11 Release VSTest. Property-container emission and public AVIF activation remain open. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs index 24e838373f..f4dac45cc7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs @@ -8,7 +8,7 @@ using System.Runtime.Intrinsics; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// -/// Builds signed AV1 residual planes from source and prediction samples. +/// Builds signed AV1 residual planes and measures sample-domain error. /// internal static partial class Av1ResidualBuilder { @@ -56,6 +56,44 @@ internal static partial class Av1ResidualBuilder int height) => Subtract(source, sourceStride, prediction, predictionStride, residual, residualStride, width, height); + /// + /// Calculates the exact squared error between strided 8-bit sample planes. + /// + /// The source samples. + /// The source row stride. + /// The prediction or reconstruction samples. + /// The prediction row stride. + /// The number of samples per row. + /// The number of rows. + /// The sum of squared sample differences. + public static long SumSquaredError( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + int width, + int height) + => SumSquaredError(source, sourceStride, prediction, predictionStride, width, height); + + /// + /// Calculates the exact squared error between strided high-bit-depth sample planes. + /// + /// The source samples. + /// The source row stride. + /// The prediction or reconstruction samples. + /// The prediction row stride. + /// The number of samples per row. + /// The number of rows. + /// The sum of squared sample differences. + public static long SumSquaredError( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + int width, + int height) + => SumSquaredError(source, sourceStride, prediction, predictionStride, width, height); + /// /// Sums the squares of a contiguous signed residual block. /// @@ -76,10 +114,7 @@ internal static partial class Av1ResidualBuilder for (; vectorCount > 0; vectorCount--, offset += Vector512.Count) { Vector512 values = Unsafe.As>(ref Unsafe.Add(ref residualBase, offset)); - Vector512 lower = Vector512.WidenLower(values); - Vector512 upper = Vector512.WidenUpper(values); - sum += Vector512.Sum(lower * lower); - sum += Vector512.Sum(upper * upper); + sum += SumSquares(values); } } @@ -90,10 +125,7 @@ internal static partial class Av1ResidualBuilder for (; vectorCount > 0; vectorCount--, offset += Vector256.Count) { Vector256 values = Unsafe.As>(ref Unsafe.Add(ref residualBase, offset)); - Vector256 lower = Vector256.WidenLower(values); - Vector256 upper = Vector256.WidenUpper(values); - sum += Vector256.Sum(lower * lower); - sum += Vector256.Sum(upper * upper); + sum += SumSquares(values); } } @@ -104,10 +136,7 @@ internal static partial class Av1ResidualBuilder for (; vectorCount > 0; vectorCount--, offset += Vector128.Count) { Vector128 values = Unsafe.As>(ref Unsafe.Add(ref residualBase, offset)); - Vector128 lower = Vector128.WidenLower(values); - Vector128 upper = Vector128.WidenUpper(values); - sum += Vector128.Sum(lower * lower); - sum += Vector128.Sum(upper * upper); + sum += SumSquares(values); } } @@ -120,6 +149,103 @@ internal static partial class Av1ResidualBuilder return sum; } + private static long SumSquaredError( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + int width, + int height) + where TSample : unmanaged + where TOperator : struct, IResidualOperator + { + long sum = 0; + for (int y = 0; y < height; y++) + { + ReadOnlySpan sourceRow = source.Slice(y * sourceStride, width); + ReadOnlySpan predictionRow = prediction.Slice(y * predictionStride, width); + ref TSample sourceBase = ref MemoryMarshal.GetReference(sourceRow); + ref TSample predictionBase = ref MemoryMarshal.GetReference(predictionRow); + int x = 0; + + // Descending hardware widths consume every complete vector before the scalar tail. Byte + // subtraction produces two widened residual vectors; high-bit-depth subtraction produces one. + if (Vector512.IsHardwareAccelerated) + { + nuint vectorCount = sourceRow.Vector512Count(); + + for (; vectorCount > 0; vectorCount--, x += Vector512.Count) + { + Vector512 sourceVector = + Unsafe.As>(ref Unsafe.Add(ref sourceBase, x)); + + Vector512 predictionVector = + Unsafe.As>(ref Unsafe.Add(ref predictionBase, x)); + + Vector512 lower = TOperator.Subtract(sourceVector, predictionVector, out Vector512 upper); + sum += SumSquares(lower); + if (Vector512.Count != Vector512.Count) + { + sum += SumSquares(upper); + } + } + } + + if (Vector256.IsHardwareAccelerated) + { + nuint vectorCount = sourceRow[x..].Vector256Count(); + + for (; vectorCount > 0; vectorCount--, x += Vector256.Count) + { + Vector256 sourceVector = + Unsafe.As>(ref Unsafe.Add(ref sourceBase, x)); + + Vector256 predictionVector = + Unsafe.As>(ref Unsafe.Add(ref predictionBase, x)); + + Vector256 lower = TOperator.Subtract(sourceVector, predictionVector, out Vector256 upper); + sum += SumSquares(lower); + if (Vector256.Count != Vector256.Count) + { + sum += SumSquares(upper); + } + } + } + + if (Vector128.IsHardwareAccelerated) + { + nuint vectorCount = sourceRow[x..].Vector128Count(); + + for (; vectorCount > 0; vectorCount--, x += Vector128.Count) + { + Vector128 sourceVector = + Unsafe.As>(ref Unsafe.Add(ref sourceBase, x)); + + Vector128 predictionVector = + Unsafe.As>(ref Unsafe.Add(ref predictionBase, x)); + + Vector128 lower = TOperator.Subtract(sourceVector, predictionVector, out Vector128 upper); + sum += SumSquares(lower); + if (Vector128.Count != Vector128.Count) + { + sum += SumSquares(upper); + } + } + } + + for (; x < width; x++) + { + int difference = TOperator.Subtract( + Unsafe.Add(ref sourceBase, x), + Unsafe.Add(ref predictionBase, x)); + + sum += difference * difference; + } + } + + return sum; + } + private static void Subtract( ReadOnlySpan source, int sourceStride, @@ -211,4 +337,28 @@ internal static partial class Av1ResidualBuilder } } } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static long SumSquares(Vector128 values) + { + Vector128 lower = Vector128.WidenLower(values); + Vector128 upper = Vector128.WidenUpper(values); + return (long)Vector128.Sum(lower * lower) + Vector128.Sum(upper * upper); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static long SumSquares(Vector256 values) + { + Vector256 lower = Vector256.WidenLower(values); + Vector256 upper = Vector256.WidenUpper(values); + return (long)Vector256.Sum(lower * lower) + Vector256.Sum(upper * upper); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static long SumSquares(Vector512 values) + { + Vector512 lower = Vector512.WidenLower(values); + Vector512 upper = Vector512.WidenUpper(values); + return (long)Vector512.Sum(lower * lower) + Vector512.Sum(upper * upper); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs index ddcaceb4e3..b2fecd255a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs @@ -145,17 +145,15 @@ internal static class Av1TransformBlockEncoder plane, ref state); - Av1ResidualBuilder.Subtract( + long distortion = Av1ResidualBuilder.SumSquaredError( sourceSamples, source.Stride, reconstruction, width, - workspace.Residual, - width, width, height); - return Av1ResidualBuilder.SumSquares(workspace.Residual[..transformSize.GetSize2d()]) << 4; + return distortion << 4; } /// @@ -199,16 +197,16 @@ internal static class Av1TransformBlockEncoder int sampleCount = transformSize.GetSize2d(); ReadOnlySpan sourceSamples = GetPlaneSpan(source, blockOrigin); - // Each transform trial mutates reconstruction and residual scratch, so restore both prepared inputs. + // Each transform trial overwrites reconstruction but consumes the prepared residual read-only. // Row copies preserve a larger candidate surface without materializing a second compact block. for (int row = 0; row < height; row++) { prediction.Slice(row * width, width).CopyTo(reconstruction.Slice(row * reconstructionStride, width)); } - residual[..sampleCount].CopyTo(workspace.Residual); EncodeLossy( workspace, + residual[..sampleCount], quantizedCoefficients, transformSize, transformType, @@ -232,17 +230,15 @@ internal static class Av1TransformBlockEncoder workspace.TransformWorkspace); } - Av1ResidualBuilder.Subtract( + long distortion = Av1ResidualBuilder.SumSquaredError( sourceSamples, source.Stride, reconstruction, reconstructionStride, - workspace.Residual, - width, width, height); - return Av1ResidualBuilder.SumSquares(workspace.Residual[..sampleCount]) << 4; + return distortion << 4; } /// @@ -322,17 +318,15 @@ internal static class Av1TransformBlockEncoder } // Final distortion is measured against the samples a decoder reconstructs, not the unquantized predictor. - Av1ResidualBuilder.Subtract( + long distortion = Av1ResidualBuilder.SumSquaredError( sourceSamples, source.Stride, reconstruction, width, - workspace.Residual, - width, width, height); - return Av1ResidualBuilder.SumSquares(workspace.Residual[..transformSize.GetSize2d()]) << 4; + return distortion << 4; } /// @@ -470,17 +464,14 @@ internal static class Av1TransformBlockEncoder bitDepth, ref state); - Av1ResidualBuilder.Subtract( + long distortion = Av1ResidualBuilder.SumSquaredError( sourceSamples, source.Stride, reconstruction, width, - workspace.Residual, - width, width, height); - long distortion = Av1ResidualBuilder.SumSquares(workspace.Residual[..transformSize.GetSize2d()]); int shift = (bitDepth.GetBitCount() - 8) * 2; long normalizedDistortion = shift == 0 ? distortion @@ -532,16 +523,16 @@ internal static class Av1TransformBlockEncoder int sampleCount = transformSize.GetSize2d(); ReadOnlySpan sourceSamples = GetPlaneSpan(source, blockOrigin); - // Each transform trial mutates reconstruction and residual scratch, so restore both prepared inputs. + // Each transform trial overwrites reconstruction but consumes the prepared residual read-only. // Row copies preserve a larger candidate surface without materializing a second compact block. for (int row = 0; row < height; row++) { prediction.Slice(row * width, width).CopyTo(reconstruction.Slice(row * reconstructionStride, width)); } - residual[..sampleCount].CopyTo(workspace.Residual); EncodeLossy( workspace, + residual[..sampleCount], quantizedCoefficients, transformSize, transformType, @@ -566,17 +557,14 @@ internal static class Av1TransformBlockEncoder workspace.TransformWorkspace); } - Av1ResidualBuilder.Subtract( + long distortion = Av1ResidualBuilder.SumSquaredError( sourceSamples, source.Stride, reconstruction, reconstructionStride, - workspace.Residual, - width, width, height); - long distortion = Av1ResidualBuilder.SumSquares(workspace.Residual[..sampleCount]); int shift = (bitDepth.GetBitCount() - 8) * 2; long normalizedDistortion = shift == 0 ? distortion @@ -672,17 +660,14 @@ internal static class Av1TransformBlockEncoder workspace.TransformWorkspace); } - Av1ResidualBuilder.Subtract( + long distortion = Av1ResidualBuilder.SumSquaredError( sourceSamples, source.Stride, reconstruction, width, - workspace.Residual, - width, width, height); - long distortion = Av1ResidualBuilder.SumSquares(workspace.Residual[..transformSize.GetSize2d()]); int shift = (bitDepth.GetBitCount() - 8) * 2; long normalizedDistortion = shift == 0 ? distortion @@ -1047,16 +1032,39 @@ internal static class Av1TransformBlockEncoder int acDeltaQ, Av1BitDepth bitDepth, ref Av1EncoderTransformBlockState state) + => EncodeLossy( + workspace, + workspace.Residual, + quantizedCoefficients, + transformSize, + transformType, + qIndex, + dcDeltaQ, + acDeltaQ, + bitDepth, + ref state); + + private static void EncodeLossy( + Av1EncoderBlockWorkspace workspace, + ReadOnlySpan residual, + Span quantizedCoefficients, + Av1TransformSize transformSize, + Av1TransformType transformType, + int qIndex, + int dcDeltaQ, + int acDeltaQ, + Av1BitDepth bitDepth, + ref Av1EncoderTransformBlockState state) { int coefficientCount = transformSize.GetAdjusted().GetSize2d(); Span transformed = workspace.TransformCoefficients[..coefficientCount]; Span quantized = quantizedCoefficients[..coefficientCount]; Span dequantized = workspace.DequantizedCoefficients[..coefficientCount]; - // The encoder keeps transformed, quantized, and reconstructed coefficients separate because mode decision - // consumes all three while only the quantized values survive in the frame coefficient owner. + // The forward transform reads the prepared residual without changing it, so every type candidate can + // reuse one source-minus-prediction block. Separate coefficient spans preserve each later representation. Av1ForwardTransformer.Transform2d( - workspace.Residual, + residual, transformed, (uint)transformSize.GetWidth(), transformType, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs index 6a97178e2c..4b58919818 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs @@ -31,7 +31,7 @@ internal static partial class Av1ForwardTransformer /// The source sample bit depth. /// The reusable workspace owned by the containing encode operation. public static void Transform2d( - Span input, + ReadOnlySpan input, Span coefficients, uint stride, Av1TransformType transformType, @@ -55,7 +55,7 @@ internal static partial class Av1ForwardTransformer /// The resolved transform functions, shifts, and axis orientation. /// The reusable transform workspace. private static void DispatchColumn( - Span input, + ReadOnlySpan input, Span coefficients, uint stride, int bitDepth, @@ -116,7 +116,7 @@ internal static partial class Av1ForwardTransformer /// The resolved transform functions, shifts, and axis orientation. /// The reusable transform workspace. private static void DispatchRow( - Span input, + ReadOnlySpan input, Span coefficients, uint stride, int bitDepth, @@ -179,7 +179,7 @@ internal static partial class Av1ForwardTransformer /// The resolved transform functions, shifts, and axis orientation. /// The reusable transform workspace. private static void Transform2d( - Span input, + ReadOnlySpan input, Span coefficients, uint stride, int bitDepth, @@ -210,7 +210,7 @@ internal static partial class Av1ForwardTransformer /// The resolved transform functions, shifts, and axis orientation. /// The reusable transform workspace. private static void TransformPacked( - Span input, + ReadOnlySpan input, Span coefficients, uint stride, ref Av1Transform2dFlipConfiguration config, @@ -324,7 +324,7 @@ internal static partial class Av1ForwardTransformer /// The resolved transform functions, shifts, and axis orientation. /// The reusable transform workspace. private static void TransformExpanded( - Span input, + ReadOnlySpan input, Span coefficients, uint stride, ref Av1Transform2dFlipConfiguration config, @@ -395,7 +395,7 @@ internal static partial class Av1ForwardTransformer /// Whether to reverse the input row order. /// Whether to reverse the samples within each row. private static void LoadPacked( - Span input, + ReadOnlySpan input, uint inputStride, ref short destination, int destinationStride, @@ -476,7 +476,7 @@ internal static partial class Av1ForwardTransformer /// Whether to reverse the input row order. /// Whether to reverse the samples within each row. private static void LoadExpanded( - Span input, + ReadOnlySpan input, uint inputStride, ref int destination, int destinationStride, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs index 5d5e497857..c101284760 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs @@ -106,7 +106,7 @@ public class Av1ResidualBuilderTests => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateSumSquares, ResidualConfigurations); /// - /// Verifies that repeated maximum-transform residual construction uses only caller-owned buffers. + /// Verifies that repeated maximum-transform residual and squared-error processing uses only caller-owned buffers. /// [Fact] public void ResidualConstructionDoesNotAllocate() @@ -124,15 +124,34 @@ public class Av1ResidualBuilderTests Av1ResidualBuilder.Subtract(source, width, prediction, width, residual, width, width, height); Av1ResidualBuilder.Subtract(highBitDepthSource, width, highBitDepthPrediction, width, highBitDepthResidual, width, width, height); - + _ = Av1ResidualBuilder.SumSquaredError(source, width, prediction, width, width, height); + _ = Av1ResidualBuilder.SumSquaredError( + highBitDepthSource, + width, + highBitDepthPrediction, + width, + width, + height); + + long sum = 0; long before = GC.GetAllocatedBytesForCurrentThread(); for (int iteration = 0; iteration < 32; iteration++) { Av1ResidualBuilder.Subtract(source, width, prediction, width, residual, width, width, height); Av1ResidualBuilder.Subtract(highBitDepthSource, width, highBitDepthPrediction, width, highBitDepthResidual, width, width, height); + sum += Av1ResidualBuilder.SumSquaredError(source, width, prediction, width, width, height); + sum += Av1ResidualBuilder.SumSquaredError( + highBitDepthSource, + width, + highBitDepthPrediction, + width, + width, + height); } - Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + long allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(sum > 0); + Assert.Equal(0, allocated); } private static void ValidateSumSquares() @@ -186,6 +205,26 @@ public class Av1ResidualBuilderTests Av1ResidualBuilder.Subtract(sourcePlane, sourceStride, predictionPlane, predictionStride, actualPlane, residualStride, width, height); Assert.Equal(expected, actual); + + long expectedSquaredError = 0; + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + int difference = sourcePlane[(y * sourceStride) + x] - predictionPlane[(y * predictionStride) + x]; + expectedSquaredError += difference * difference; + } + } + + Assert.Equal( + expectedSquaredError, + Av1ResidualBuilder.SumSquaredError( + sourcePlane, + sourceStride, + predictionPlane, + predictionStride, + width, + height)); } private static void ValidateUInt16Residuals(int maximumSample) @@ -215,6 +254,26 @@ public class Av1ResidualBuilderTests Av1ResidualBuilder.Subtract(sourcePlane, sourceStride, predictionPlane, predictionStride, actualPlane, residualStride, width, height); Assert.Equal(expected, actual); + + long expectedSquaredError = 0; + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + int difference = sourcePlane[(y * sourceStride) + x] - predictionPlane[(y * predictionStride) + x]; + expectedSquaredError += difference * difference; + } + } + + Assert.Equal( + expectedSquaredError, + Av1ResidualBuilder.SumSquaredError( + sourcePlane, + sourceStride, + predictionPlane, + predictionStride, + width, + height)); } private static void FillBytePlanes(Span source, int sourceStride, Span prediction, int predictionStride, int width, int height)