diff --git a/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs b/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs index 57c202918..942a80ab7 100644 --- a/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs +++ b/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs @@ -140,98 +140,12 @@ internal static class AverageFilter /// The sum of the total variance of the filtered row. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Encode(ReadOnlySpan scanline, ReadOnlySpan previousScanline, Span result, uint bytesPerPixel, out int sum) - { - DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); - DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result)); - - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); - sum = 0; - - // Average(x) = Raw(x) - floor((Raw(x-bpp)+Prior(x))/2) - resultBaseRef = (byte)FilterType.Average; - - nuint x = 0; - for (; x < bytesPerPixel; /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - byte above = Unsafe.Add(ref prevBaseRef, x); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = (byte)(scan - (above >> 1)); - sum += Numerics.Abs(unchecked((sbyte)res)); - } - - if (Avx2.IsSupported) - { - Vector256 zero = Vector256.Zero; - Vector256 sumAccumulator = Vector256.Zero; - Vector256 allBitsSet = Avx2.CompareEqual(sumAccumulator, sumAccumulator).AsByte(); - - for (nuint xLeft = x - bytesPerPixel; (int)x <= scanline.Length - Vector256.Count; xLeft += (uint)Vector256.Count) - { - Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector256 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); - Vector256 above = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, x)); - - Vector256 avg = Avx2.Xor(Avx2.Average(Avx2.Xor(left, allBitsSet), Avx2.Xor(above, allBitsSet)), allBitsSet); - Vector256 res = Avx2.Subtract(scan, avg); - - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector256.Count; - - sumAccumulator = Avx2.Add(sumAccumulator, Avx2.SumAbsoluteDifferences(Avx2.Abs(res.AsSByte()), zero).AsInt32()); - } - - sum += Numerics.EvenReduceSum(sumAccumulator); - } - else if (Sse2.IsSupported) - { - Vector128 zero = Vector128.Zero; - Vector128 sumAccumulator = Vector128.Zero; - Vector128 allBitsSet = Sse2.CompareEqual(sumAccumulator, sumAccumulator).AsByte(); - - for (nuint xLeft = x - bytesPerPixel; (int)x <= scanline.Length - Vector128.Count; xLeft += (uint)Vector128.Count) - { - Vector128 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector128 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); - Vector128 above = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, x)); - - Vector128 avg = Sse2.Xor(Sse2.Average(Sse2.Xor(left, allBitsSet), Sse2.Xor(above, allBitsSet)), allBitsSet); - Vector128 res = Sse2.Subtract(scan, avg); - - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector128.Count; - - Vector128 absRes; - if (Ssse3.IsSupported) - { - absRes = Ssse3.Abs(res.AsSByte()); - } - else - { - Vector128 mask = Sse2.CompareGreaterThan(zero.AsSByte(), res.AsSByte()); - absRes = Sse2.Xor(Sse2.Add(res.AsSByte(), mask), mask).AsByte(); - } - - sumAccumulator = Sse2.Add(sumAccumulator, Sse2.SumAbsoluteDifferences(absRes, zero).AsInt32()); - } - - sum += Numerics.EvenReduceSum(sumAccumulator); - } - - for (nuint xLeft = x - bytesPerPixel; x < (uint)scanline.Length; ++xLeft /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - byte left = Unsafe.Add(ref scanBaseRef, xLeft); - byte above = Unsafe.Add(ref prevBaseRef, x); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = (byte)(scan - Average(left, above)); - sum += Numerics.Abs(unchecked((sbyte)res)); - } - } + => PngFilterEncoder.Encode( + scanline, + previousScanline, + result, + bytesPerPixel, + out sum); /// /// Calculates the average value of two bytes diff --git a/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs b/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs new file mode 100644 index 000000000..1113a5df6 --- /dev/null +++ b/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs @@ -0,0 +1,524 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace SixLabors.ImageSharp.Formats.Png.Filters; + +/// +/// Defines the scalar and SIMD mappings used by the shared PNG filter traversal. +/// +internal interface IPngFilterOperator +{ + /// + /// Gets the filter type written to the leading result byte. + /// + static abstract FilterType Type { get; } + + /// + /// Gets a value indicating whether the predictor reads the left component. + /// + static abstract bool UsesLeft { get; } + + /// + /// Gets a value indicating whether the predictor reads the above component. + /// + static abstract bool UsesAbove { get; } + + /// + /// Gets a value indicating whether the predictor reads the upper-left component. + /// + static abstract bool UsesUpperLeft { get; } + + /// + /// Filters one byte from its PNG neighborhood. + /// + /// The component being filtered. + /// The corresponding component in the preceding pixel. + /// The corresponding component in the preceding scanline. + /// The preceding component in the preceding scanline. + /// The filtered residual. + static abstract byte Invoke(byte scan, byte left, byte above, byte upperLeft); + + /// + /// Filters sixteen byte lanes from their PNG neighborhoods. + /// + /// The components being filtered. + /// The corresponding components in the preceding pixels. + /// The corresponding components in the preceding scanline. + /// The preceding components in the preceding scanline. + /// The filtered residuals. + static abstract Vector128 Invoke( + Vector128 scan, + Vector128 left, + Vector128 above, + Vector128 upperLeft); + + /// + /// Filters thirty-two byte lanes from their PNG neighborhoods. + /// + /// The components being filtered. + /// The corresponding components in the preceding pixels. + /// The corresponding components in the preceding scanline. + /// The preceding components in the preceding scanline. + /// The filtered residuals. + static abstract Vector256 Invoke( + Vector256 scan, + Vector256 left, + Vector256 above, + Vector256 upperLeft); + + /// + /// Filters sixty-four byte lanes from their PNG neighborhoods. + /// + /// The components being filtered. + /// The corresponding components in the preceding pixels. + /// The corresponding components in the preceding scanline. + /// The preceding components in the preceding scanline. + /// The filtered residuals. + static abstract Vector512 Invoke( + Vector512 scan, + Vector512 left, + Vector512 above, + Vector512 upperLeft); +} + +/// +/// Maps each component to its difference from the corresponding component in the preceding pixel. +/// +internal readonly struct SubFilterOperator : IPngFilterOperator +{ + /// + public static FilterType Type => FilterType.Sub; + + /// + public static bool UsesLeft => true; + + /// + public static bool UsesAbove => false; + + /// + public static bool UsesUpperLeft => false; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static byte Invoke(byte scan, byte left, byte above, byte upperLeft) => (byte)(scan - left); + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector128 Invoke( + Vector128 scan, + Vector128 left, + Vector128 above, + Vector128 upperLeft) + => scan - left; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector256 Invoke( + Vector256 scan, + Vector256 left, + Vector256 above, + Vector256 upperLeft) + => scan - left; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector512 Invoke( + Vector512 scan, + Vector512 left, + Vector512 above, + Vector512 upperLeft) + => scan - left; +} + +/// +/// Maps each component to its difference from the component directly above it. +/// +internal readonly struct UpFilterOperator : IPngFilterOperator +{ + /// + public static FilterType Type => FilterType.Up; + + /// + public static bool UsesLeft => false; + + /// + public static bool UsesAbove => true; + + /// + public static bool UsesUpperLeft => false; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static byte Invoke(byte scan, byte left, byte above, byte upperLeft) => (byte)(scan - above); + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector128 Invoke( + Vector128 scan, + Vector128 left, + Vector128 above, + Vector128 upperLeft) + => scan - above; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector256 Invoke( + Vector256 scan, + Vector256 left, + Vector256 above, + Vector256 upperLeft) + => scan - above; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector512 Invoke( + Vector512 scan, + Vector512 left, + Vector512 above, + Vector512 upperLeft) + => scan - above; +} + +/// +/// Maps each component to its difference from the truncated average of its left and above neighbors. +/// +internal readonly struct AverageFilterOperator : IPngFilterOperator +{ + /// + public static FilterType Type => FilterType.Average; + + /// + public static bool UsesLeft => true; + + /// + public static bool UsesAbove => true; + + /// + public static bool UsesUpperLeft => false; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static byte Invoke(byte scan, byte left, byte above, byte upperLeft) + => (byte)(scan - ((left + above) >> 1)); + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector128 Invoke( + Vector128 scan, + Vector128 left, + Vector128 above, + Vector128 upperLeft) + { + Vector128 average; + + if (Sse2.IsSupported) + { + // PAVG rounds upward. Complementing both inputs and the result converts + // that rounding into the floor((left + above) / 2) required by PNG. + average = ~Sse2.Average(~left, ~above); + } + else if (AdvSimd.IsSupported) + { + // ARM's halving add truncates directly and therefore needs no correction. + average = AdvSimd.FusedAddHalving(left, above); + } + else + { + // Portable 128-bit backends use the carry-free average identity. Shared + // bits supply the integer part while differing bits supply half the remainder. + average = (left & above) + Vector128.ShiftRightLogical(left ^ above, 1); + } + + return scan - average; + } + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector256 Invoke( + Vector256 scan, + Vector256 left, + Vector256 above, + Vector256 upperLeft) + => scan - ~Avx2.Average(~left, ~above); + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector512 Invoke( + Vector512 scan, + Vector512 left, + Vector512 above, + Vector512 upperLeft) + => scan - ~Avx512BW.Average(~left, ~above); +} + +/// +/// Maps each component to its difference from the nearest Paeth neighbor. +/// +internal readonly struct PaethFilterOperator : IPngFilterOperator +{ + /// + public static FilterType Type => FilterType.Paeth; + + /// + public static bool UsesLeft => true; + + /// + public static bool UsesAbove => true; + + /// + public static bool UsesUpperLeft => true; + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static byte Invoke(byte scan, byte left, byte above, byte upperLeft) + { + int p = left + above - upperLeft; + int distanceLeft = Numerics.Abs(p - left); + int distanceAbove = Numerics.Abs(p - above); + int distanceUpperLeft = Numerics.Abs(p - upperLeft); + + // PNG resolves equal distances in left, above, upper-left order. + byte predictor = distanceLeft <= distanceAbove && distanceLeft <= distanceUpperLeft + ? left + : distanceAbove <= distanceUpperLeft ? above : upperLeft; + + return (byte)(scan - predictor); + } + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector128 Invoke( + Vector128 scan, + Vector128 left, + Vector128 above, + Vector128 upperLeft) + { + Vector128 predictor = Predict(left, above, upperLeft); + return scan - predictor; + } + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector256 Invoke( + Vector256 scan, + Vector256 left, + Vector256 above, + Vector256 upperLeft) + { + Vector256 predictor = Predict(left, above, upperLeft); + return scan - predictor; + } + + /// + [MethodImpl(InliningOptions.AlwaysInline)] + public static Vector512 Invoke( + Vector512 scan, + Vector512 left, + Vector512 above, + Vector512 upperLeft) + { + Vector512 predictor = Predict(left, above, upperLeft); + return scan - predictor; + } + + /// + /// Selects the nearest Paeth neighbor for sixteen independent byte lanes. + /// + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector128 Predict( + Vector128 left, + Vector128 above, + Vector128 upperLeft) + { + Vector128 aboveMinusUpper = SubtractSaturate(above, upperLeft); + Vector128 leftMinusUpper = SubtractSaturate(left, upperLeft); + Vector128 distanceLeft = SubtractSaturate(upperLeft, above) | aboveMinusUpper; + Vector128 distanceAbove = SubtractSaturate(upperLeft, left) | leftMinusUpper; + + return SelectPredictor( + left, + above, + upperLeft, + aboveMinusUpper, + leftMinusUpper, + distanceLeft, + distanceAbove); + } + + /// + /// Selects the nearest Paeth neighbor for thirty-two independent byte lanes. + /// + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector256 Predict( + Vector256 left, + Vector256 above, + Vector256 upperLeft) + { + Vector256 aboveMinusUpper = Avx2.SubtractSaturate(above, upperLeft); + Vector256 leftMinusUpper = Avx2.SubtractSaturate(left, upperLeft); + Vector256 distanceLeft = Avx2.SubtractSaturate(upperLeft, above) | aboveMinusUpper; + Vector256 distanceAbove = Avx2.SubtractSaturate(upperLeft, left) | leftMinusUpper; + + return SelectPredictor( + left, + above, + upperLeft, + aboveMinusUpper, + leftMinusUpper, + distanceLeft, + distanceAbove); + } + + /// + /// Selects the nearest Paeth neighbor for sixty-four independent byte lanes. + /// + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector512 Predict( + Vector512 left, + Vector512 above, + Vector512 upperLeft) + { + Vector512 aboveMinusUpper = Avx512BW.SubtractSaturate(above, upperLeft); + Vector512 leftMinusUpper = Avx512BW.SubtractSaturate(left, upperLeft); + Vector512 distanceLeft = Avx512BW.SubtractSaturate(upperLeft, above) | aboveMinusUpper; + Vector512 distanceAbove = Avx512BW.SubtractSaturate(upperLeft, left) | leftMinusUpper; + + return SelectPredictor( + left, + above, + upperLeft, + aboveMinusUpper, + leftMinusUpper, + distanceLeft, + distanceAbove); + } + + /// + /// Applies Paeth distance and tie-breaking rules to sixteen lanes. + /// + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector128 SelectPredictor( + Vector128 left, + Vector128 above, + Vector128 upperLeft, + Vector128 aboveMinusUpper, + Vector128 leftMinusUpper, + Vector128 distanceLeft, + Vector128 distanceAbove) + { + Vector128 sameDirection = Vector128.Equals( + Vector128.Equals(aboveMinusUpper, Vector128.Zero), + Vector128.Equals(leftMinusUpper, Vector128.Zero)); + + Vector128 distanceUpper = sameDirection + | SubtractSaturate(distanceAbove, distanceLeft) + | SubtractSaturate(distanceLeft, distanceAbove); + + Vector128 minimumAboveUpper = Vector128.Min(distanceUpper, distanceAbove); + Vector128 aboveOrUpper = Vector128.ConditionalSelect( + Vector128.Equals(minimumAboveUpper, distanceAbove), + above, + upperLeft); + + // Applying the left comparison last preserves PNG's left-first tie rule. + return Vector128.ConditionalSelect( + Vector128.Equals(Vector128.Min(minimumAboveUpper, distanceLeft), distanceLeft), + left, + aboveOrUpper); + } + + /// + /// Applies Paeth distance and tie-breaking rules to thirty-two lanes. + /// + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector256 SelectPredictor( + Vector256 left, + Vector256 above, + Vector256 upperLeft, + Vector256 aboveMinusUpper, + Vector256 leftMinusUpper, + Vector256 distanceLeft, + Vector256 distanceAbove) + { + Vector256 sameDirection = Vector256.Equals( + Vector256.Equals(aboveMinusUpper, Vector256.Zero), + Vector256.Equals(leftMinusUpper, Vector256.Zero)); + + Vector256 distanceUpper = sameDirection + | Avx2.SubtractSaturate(distanceAbove, distanceLeft) + | Avx2.SubtractSaturate(distanceLeft, distanceAbove); + + Vector256 minimumAboveUpper = Vector256.Min(distanceUpper, distanceAbove); + Vector256 aboveOrUpper = Vector256.ConditionalSelect( + Vector256.Equals(minimumAboveUpper, distanceAbove), + above, + upperLeft); + + return Vector256.ConditionalSelect( + Vector256.Equals(Vector256.Min(minimumAboveUpper, distanceLeft), distanceLeft), + left, + aboveOrUpper); + } + + /// + /// Applies Paeth distance and tie-breaking rules to sixty-four lanes. + /// + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector512 SelectPredictor( + Vector512 left, + Vector512 above, + Vector512 upperLeft, + Vector512 aboveMinusUpper, + Vector512 leftMinusUpper, + Vector512 distanceLeft, + Vector512 distanceAbove) + { + Vector512 sameDirection = Vector512.Equals( + Vector512.Equals(aboveMinusUpper, Vector512.Zero), + Vector512.Equals(leftMinusUpper, Vector512.Zero)); + + Vector512 distanceUpper = sameDirection + | Avx512BW.SubtractSaturate(distanceAbove, distanceLeft) + | Avx512BW.SubtractSaturate(distanceLeft, distanceAbove); + + Vector512 minimumAboveUpper = Vector512.Min(distanceUpper, distanceAbove); + Vector512 aboveOrUpper = Vector512.ConditionalSelect( + Vector512.Equals(minimumAboveUpper, distanceAbove), + above, + upperLeft); + + return Vector512.ConditionalSelect( + Vector512.Equals(Vector512.Min(minimumAboveUpper, distanceLeft), distanceLeft), + left, + aboveOrUpper); + } + + /// + /// Performs an unsigned saturating subtraction using the active 128-bit instruction set. + /// + /// The minuend lanes. + /// The subtrahend lanes. + /// The saturated lane-wise differences. + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector128 SubtractSaturate(Vector128 left, Vector128 right) + { + if (Sse2.IsSupported) + { + return Sse2.SubtractSaturate(left, right); + } + + if (AdvSimd.IsSupported) + { + return AdvSimd.SubtractSaturate(left, right); + } + + // Subtracting the smaller operand produces max(left - right, 0) without + // requiring a backend-specific saturating-subtract instruction. + return left - Vector128.Min(left, right); + } +} diff --git a/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs b/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs index 59c903c1d..0216a2627 100644 --- a/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs +++ b/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -193,86 +192,12 @@ internal static class PaethFilter /// The sum of the total variance of the filtered row. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Encode(ReadOnlySpan scanline, ReadOnlySpan previousScanline, Span result, int bytesPerPixel, out int sum) - { - DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); - DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result)); - - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); - sum = 0; - - // Paeth(x) = Raw(x) - PaethPredictor(Raw(x-bpp), Prior(x), Prior(x - bpp)) - resultBaseRef = (byte)FilterType.Paeth; - - nuint x = 0; - for (; x < (uint)bytesPerPixel; /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - byte above = Unsafe.Add(ref prevBaseRef, x); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = (byte)(scan - PaethPredictor(0, above, 0)); - sum += Numerics.Abs(unchecked((sbyte)res)); - } - - if (Avx2.IsSupported) - { - Vector256 zero = Vector256.Zero; - Vector256 sumAccumulator = Vector256.Zero; - - for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= scanline.Length - Vector256.Count; xLeft += (uint)Vector256.Count) - { - Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector256 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); - Vector256 above = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, x)); - Vector256 upperLeft = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, xLeft)); - - Vector256 res = Avx2.Subtract(scan, PaethPredictor(left, above, upperLeft)); - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector256.Count; - - sumAccumulator = Avx2.Add(sumAccumulator, Avx2.SumAbsoluteDifferences(Avx2.Abs(res.AsSByte()), zero).AsInt32()); - } - - sum += Numerics.EvenReduceSum(sumAccumulator); - } - else if (Vector.IsHardwareAccelerated) - { - Vector sumAccumulator = Vector.Zero; - - for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= scanline.Length - Vector.Count; xLeft += (uint)Vector.Count) - { - Vector scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); - Vector above = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, x)); - Vector upperLeft = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, xLeft)); - - Vector res = scan - PaethPredictor(left, above, upperLeft); - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector.Count; - - Numerics.Accumulate(ref sumAccumulator, Vector.AsVectorByte(Vector.Abs(Vector.AsVectorSByte(res)))); - } - - for (int i = 0; i < Vector.Count; i++) - { - sum += (int)sumAccumulator[i]; - } - } - - for (nuint xLeft = x - (uint)bytesPerPixel; (int)x < scanline.Length; ++xLeft /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - byte left = Unsafe.Add(ref scanBaseRef, xLeft); - byte above = Unsafe.Add(ref prevBaseRef, x); - byte upperLeft = Unsafe.Add(ref prevBaseRef, xLeft); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = (byte)(scan - PaethPredictor(left, above, upperLeft)); - sum += Numerics.Abs(unchecked((sbyte)res)); - } - } + => PngFilterEncoder.Encode( + scanline, + previousScanline, + result, + (uint)bytesPerPixel, + out sum); /// /// Computes a simple linear function of the three neighboring pixels (left, above, upper left), then chooses @@ -304,70 +229,4 @@ internal static class PaethFilter return upperLeft; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 PaethPredictor(Vector256 left, Vector256 above, Vector256 upleft) - { - Vector256 zero = Vector256.Zero; - - // Here, we refactor pa = abs(p - left) = abs(left + above - upleft - left) - // to pa = abs(above - upleft). Same deal for pb. - // Using saturated subtraction, if the result is negative, the output is zero. - // If we subtract in both directions and `or` the results, only one can be - // non-zero, so we end up with the absolute value. - Vector256 sac = Avx2.SubtractSaturate(above, upleft); - Vector256 sbc = Avx2.SubtractSaturate(left, upleft); - Vector256 pa = Avx2.Or(Avx2.SubtractSaturate(upleft, above), sac); - Vector256 pb = Avx2.Or(Avx2.SubtractSaturate(upleft, left), sbc); - - // pc = abs(left + above - upleft - upleft), or abs(left - upleft + above - upleft). - // We've already calculated left - upleft and above - upleft in `sac` and `sbc`. - // If they are both negative or both positive, the absolute value of their - // sum can't possibly be less than `pa` or `pb`, so we'll never use the value. - // We make a mask that sets the value to 255 if they either both got - // saturated to zero or both didn't. Then we calculate the absolute value - // of their difference using saturated subtract and `or`, same as before, - // keeping the value only where the mask isn't set. - Vector256 pm = Avx2.CompareEqual(Avx2.CompareEqual(sac, zero), Avx2.CompareEqual(sbc, zero)); - Vector256 pc = Avx2.Or(pm, Avx2.Or(Avx2.SubtractSaturate(pb, pa), Avx2.SubtractSaturate(pa, pb))); - - // Finally, blend the values together. We start with `upleft` and overwrite on - // tied values so that the `left`, `above`, `upleft` precedence is preserved. - Vector256 minbc = Avx2.Min(pc, pb); - Vector256 resbc = Avx2.BlendVariable(upleft, above, Avx2.CompareEqual(minbc, pb)); - return Avx2.BlendVariable(resbc, left, Avx2.CompareEqual(Avx2.Min(minbc, pa), pa)); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector PaethPredictor(Vector left, Vector above, Vector upperLeft) - { - Vector.Widen(left, out Vector a1, out Vector a2); - Vector.Widen(above, out Vector b1, out Vector b2); - Vector.Widen(upperLeft, out Vector c1, out Vector c2); - - Vector p1 = PaethPredictor(Vector.AsVectorInt16(a1), Vector.AsVectorInt16(b1), Vector.AsVectorInt16(c1)); - Vector p2 = PaethPredictor(Vector.AsVectorInt16(a2), Vector.AsVectorInt16(b2), Vector.AsVectorInt16(c2)); - return Vector.AsVectorByte(Vector.Narrow(p1, p2)); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector PaethPredictor(Vector left, Vector above, Vector upperLeft) - { - Vector p = left + above - upperLeft; - Vector pa = Vector.Abs(p - left); - Vector pb = Vector.Abs(p - above); - Vector pc = Vector.Abs(p - upperLeft); - - Vector pa_pb = Vector.LessThanOrEqual(pa, pb); - Vector pa_pc = Vector.LessThanOrEqual(pa, pc); - Vector pb_pc = Vector.LessThanOrEqual(pb, pc); - - return Vector.ConditionalSelect( - condition: Vector.BitwiseAnd(pa_pb, pa_pc), - left: left, - right: Vector.ConditionalSelect( - condition: pb_pc, - left: above, - right: upperLeft)); - } } diff --git a/src/ImageSharp/Formats/Png/Filters/PngFilterEncoder.cs b/src/ImageSharp/Formats/Png/Filters/PngFilterEncoder.cs new file mode 100644 index 000000000..4c298e0fc --- /dev/null +++ b/src/ImageSharp/Formats/Png/Filters/PngFilterEncoder.cs @@ -0,0 +1,228 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace SixLabors.ImageSharp.Formats.Png.Filters; + +/// +/// Applies PNG filter operators while accumulating the absolute signed residuals used for filter selection. +/// +internal static class PngFilterEncoder +{ + /// + /// Maps a scanline through a filter operator, writes the residuals, and reduces their total variance. + /// + /// The PNG predictor selected for this closed traversal. + /// The scanline to encode. + /// The preceding scanline. + /// The destination including its leading filter byte. + /// The distance to the corresponding component in the preceding pixel. + /// The sum of the absolute signed residuals. + // Inlining closes every static interface call over TOperator. The JIT can then remove + // source loads ignored by simpler predictors and specialize the active register widths. + [MethodImpl(InliningOptions.AlwaysInline)] + public static void Encode( + ReadOnlySpan scanline, + ReadOnlySpan previousScanline, + Span result, + uint bytesPerPixel, + out int sum) + where TOperator : struct, IPngFilterOperator + { + DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); + DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result)); + + ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); + ref byte previousBaseRef = ref MemoryMarshal.GetReference(previousScanline); + ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); + + resultBaseRef = (byte)TOperator.Type; + sum = 0; + + nuint x = 0; + + // Components in the first pixel have no left or upper-left neighbor. Supplying + // zeroes expresses the PNG boundary rule directly through the same predictor. + for (; x < bytesPerPixel; x++) + { + byte above = TOperator.UsesAbove ? Unsafe.Add(ref previousBaseRef, x) : (byte)0; + + byte filtered = TOperator.Invoke( + Unsafe.Add(ref scanBaseRef, x), + 0, + above, + 0); + + Unsafe.Add(ref resultBaseRef, x + 1) = filtered; + sum += Numerics.Abs(unchecked((sbyte)filtered)); + } + + Vector128 sum128 = Vector128.Zero; + + // A single 512-bit register does not amortize folding its SAD accumulator. + // Leave short rows to the narrower paths, which have lower fixed reduction cost. + if (Avx512BW.IsSupported && scanline.Length - (int)x >= Vector512.Count * 2) + { + Vector512 sum512 = Vector512.Zero; + int oneRegisterFromEnd = scanline.Length - Vector512.Count; + + for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector512.Count) + { + // Each byte lane represents one independently filtered component. The + // four input vectors retain the scan/left/above/upper-left PNG layout. + // Operator usage flags are constants after generic specialization, so + // unused predictors do not retain even fault-preserving probe loads. + Vector512 left = TOperator.UsesLeft + ? Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)) + : default; + Vector512 above = TOperator.UsesAbove + ? Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)) + : default; + Vector512 upperLeft = TOperator.UsesUpperLeft + ? Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, xLeft)) + : default; + + Vector512 filtered = TOperator.Invoke( + Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)), + left, + above, + upperLeft); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered; + x += (uint)Vector512.Count; + + // PNG scores each residual as abs((sbyte)residual). VPSADBW sums eight + // byte lanes into every other 32-bit lane without widening each byte. + Vector512 absolute = Avx512BW.Abs(filtered.AsSByte()); + sum512 += Avx512BW.SumAbsoluteDifferences(absolute, Vector512.Zero).AsUInt32(); + } + + // Fold only widths that processed data. Short rows therefore avoid both + // wide accumulator initialization and an otherwise empty reduction. + Vector256 folded512 = sum512.GetLower() + sum512.GetUpper(); + sum128 += folded512.GetLower() + folded512.GetUpper(); + } + + if (Avx2.IsSupported) + { + Vector256 sum256 = Vector256.Zero; + int oneRegisterFromEnd = scanline.Length - Vector256.Count; + + for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector256.Count) + { + Vector256 left = TOperator.UsesLeft + ? Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)) + : default; + Vector256 above = TOperator.UsesAbove + ? Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)) + : default; + Vector256 upperLeft = TOperator.UsesUpperLeft + ? Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, xLeft)) + : default; + + Vector256 filtered = TOperator.Invoke( + Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)), + left, + above, + upperLeft); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered; + x += (uint)Vector256.Count; + + Vector256 absolute = Avx2.Abs(filtered.AsSByte()); + sum256 += Avx2.SumAbsoluteDifferences(absolute, Vector256.Zero).AsUInt32(); + } + + sum128 += sum256.GetLower() + sum256.GetUpper(); + } + + if (Vector128.IsHardwareAccelerated) + { + int oneRegisterFromEnd = scanline.Length - Vector128.Count; + + for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector128.Count) + { + Vector128 left = TOperator.UsesLeft + ? Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)) + : default; + Vector128 above = TOperator.UsesAbove + ? Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)) + : default; + Vector128 upperLeft = TOperator.UsesUpperLeft + ? Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, xLeft)) + : default; + + Vector128 filtered = TOperator.Invoke( + Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)), + left, + above, + upperLeft); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered; + x += (uint)Vector128.Count; + + sum128 = AccumulateAbsolute(sum128, filtered); + } + } + + sum += unchecked((int)Vector128.Sum(sum128)); + + for (nuint xLeft = x - bytesPerPixel; x < (uint)scanline.Length; xLeft++, x++) + { + byte left = TOperator.UsesLeft ? Unsafe.Add(ref scanBaseRef, xLeft) : (byte)0; + byte above = TOperator.UsesAbove ? Unsafe.Add(ref previousBaseRef, x) : (byte)0; + byte upperLeft = TOperator.UsesUpperLeft ? Unsafe.Add(ref previousBaseRef, xLeft) : (byte)0; + + byte filtered = TOperator.Invoke( + Unsafe.Add(ref scanBaseRef, x), + left, + above, + upperLeft); + + Unsafe.Add(ref resultBaseRef, x + 1) = filtered; + sum += Numerics.Abs(unchecked((sbyte)filtered)); + } + } + + /// + /// Accumulates the absolute signed values in a 128-bit residual vector. + /// + /// The four-lane unsigned accumulator. + /// The sixteen filtered byte residuals. + /// The updated accumulator. + [MethodImpl(InliningOptions.AlwaysInline)] + private static Vector128 AccumulateAbsolute(Vector128 accumulator, Vector128 residuals) + { + if (Sse2.IsSupported) + { + Vector128 absolute; + + if (Ssse3.IsSupported) + { + absolute = Ssse3.Abs(residuals.AsSByte()); + } + else + { + // SSE2 has no packed signed-byte absolute instruction. The sign mask + // implements (value + mask) XOR mask, including -128 -> 128. + Vector128 mask = Sse2.CompareGreaterThan(Vector128.Zero, residuals.AsSByte()); + absolute = Sse2.Xor(Sse2.Add(residuals.AsSByte(), mask), mask).AsByte(); + } + + return accumulator + Sse2.SumAbsoluteDifferences(absolute, Vector128.Zero).AsUInt32(); + } + + Vector128 absoluteArm = Vector128.Abs(residuals.AsSByte()).AsByte(); + (Vector128 lower16, Vector128 upper16) = Vector128.Widen(absoluteArm); + (Vector128 lower0, Vector128 lower1) = Vector128.Widen(lower16); + (Vector128 upper0, Vector128 upper1) = Vector128.Widen(upper16); + + // Four widening additions keep every byte contribution in a 32-bit lane, + // matching the x86 accumulator's overflow behavior without scalar reduction. + return accumulator + lower0 + lower1 + upper0 + upper1; + } +} diff --git a/src/ImageSharp/Formats/Png/Filters/SubFilter.cs b/src/ImageSharp/Formats/Png/Filters/SubFilter.cs index 1af4a3b72..957fb670f 100644 --- a/src/ImageSharp/Formats/Png/Filters/SubFilter.cs +++ b/src/ImageSharp/Formats/Png/Filters/SubFilter.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -110,77 +109,11 @@ internal static class SubFilter /// The bytes per pixel. /// The sum of the total variance of the filtered row. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Encode(ReadOnlySpan scanline, ReadOnlySpan result, int bytesPerPixel, out int sum) - { - DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result)); - - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); - sum = 0; - - // Sub(x) = Raw(x) - Raw(x-bpp) - resultBaseRef = (byte)FilterType.Sub; - - nuint x = 0; - for (; x < (uint)bytesPerPixel; /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = scan; - sum += Numerics.Abs(unchecked((sbyte)res)); - } - - if (Avx2.IsSupported) - { - Vector256 zero = Vector256.Zero; - Vector256 sumAccumulator = Vector256.Zero; - - for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= (scanline.Length - Vector256.Count); xLeft += (uint)Vector256.Count) - { - Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector256 prev = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); - - Vector256 res = Avx2.Subtract(scan, prev); - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector256.Count; - - sumAccumulator = Avx2.Add(sumAccumulator, Avx2.SumAbsoluteDifferences(Avx2.Abs(res.AsSByte()), zero).AsInt32()); - } - - sum += Numerics.EvenReduceSum(sumAccumulator); - } - else - if (Vector.IsHardwareAccelerated) - { - Vector sumAccumulator = Vector.Zero; - - for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= (scanline.Length - Vector.Count); xLeft += (uint)Vector.Count) - { - Vector scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector prev = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); - - Vector res = scan - prev; - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector.Count; - - Numerics.Accumulate(ref sumAccumulator, Vector.AsVectorByte(Vector.Abs(Vector.AsVectorSByte(res)))); - } - - for (int i = 0; i < Vector.Count; i++) - { - sum += (int)sumAccumulator[i]; - } - } - - for (nuint xLeft = x - (uint)bytesPerPixel; x < (uint)scanline.Length; ++xLeft /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - byte prev = Unsafe.Add(ref scanBaseRef, xLeft); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = (byte)(scan - prev); - sum += Numerics.Abs(unchecked((sbyte)res)); - } - } + public static void Encode(ReadOnlySpan scanline, Span result, int bytesPerPixel, out int sum) + => PngFilterEncoder.Encode( + scanline, + scanline, + result, + (uint)bytesPerPixel, + out sum); } diff --git a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs index d9c8e36d6..5f78833cd 100644 --- a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs +++ b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs @@ -1,11 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Numerics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.Formats.Png.Filters; @@ -40,69 +36,10 @@ internal static class UpFilter /// The sum of the total variance of the filtered row. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Encode(ReadOnlySpan scanline, ReadOnlySpan previousScanline, Span result, out int sum) - { - DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); - DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result)); - - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); - sum = 0; - - // Up(x) = Raw(x) - Prior(x) - resultBaseRef = (byte)FilterType.Up; - - nuint x = 0; - - if (Avx2.IsSupported) - { - Vector256 zero = Vector256.Zero; - Vector256 sumAccumulator = Vector256.Zero; - - for (; (int)x <= scanline.Length - Vector256.Count;) - { - Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector256 above = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, x)); - - Vector256 res = Avx2.Subtract(scan, above); - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector256.Count; - - sumAccumulator = Avx2.Add(sumAccumulator, Avx2.SumAbsoluteDifferences(Avx2.Abs(res.AsSByte()), zero).AsInt32()); - } - - sum += Numerics.EvenReduceSum(sumAccumulator); - } - else if (Vector.IsHardwareAccelerated) - { - Vector sumAccumulator = Vector.Zero; - - for (; (int)x <= scanline.Length - Vector.Count;) - { - Vector scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); - Vector above = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, x)); - - Vector res = scan - above; - Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = res; // +1 to skip filter type - x += (uint)Vector.Count; - - Numerics.Accumulate(ref sumAccumulator, Vector.AsVectorByte(Vector.Abs(Vector.AsVectorSByte(res)))); - } - - for (int i = 0; i < Vector.Count; i++) - { - sum += (int)sumAccumulator[i]; - } - } - - for (; x < (uint)scanline.Length; /* Note: ++x happens in the body to avoid one add operation */) - { - byte scan = Unsafe.Add(ref scanBaseRef, x); - byte above = Unsafe.Add(ref prevBaseRef, x); - ++x; - ref byte res = ref Unsafe.Add(ref resultBaseRef, x); - res = (byte)(scan - above); - sum += Numerics.Abs(unchecked((sbyte)res)); - } - } + => PngFilterEncoder.Encode( + scanline, + previousScanline, + result, + 0, + out sum); } diff --git a/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncode.cs b/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncode.cs new file mode 100644 index 000000000..0547376b7 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncode.cs @@ -0,0 +1,157 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Formats.Png; +using SixLabors.ImageSharp.Formats.Png.Filters; + +namespace SixLabors.ImageSharp.Benchmarks.Codecs.Png; + +/// +/// Compares the shared PNG map/reduce traversal with the filter-specific traversals it replaces. +/// +[Config(typeof(Config.Short))] +public class PngFilterEncode +{ + private const int BytesPerPixel = 4; + + private byte[] scanline; + private byte[] previousScanline; + private byte[] currentResult; + private byte[] baselineResult; + + /// + /// Gets or sets the filter evaluated by each invocation. + /// + [Params(PngFilterMethod.Sub, PngFilterMethod.Up, PngFilterMethod.Average, PngFilterMethod.Paeth)] + public PngFilterMethod Filter { get; set; } + + /// + /// Gets or sets the number of scanline bytes. + /// + [Params(64, 1024, 16384)] + public int Count { get; set; } + + /// + /// Creates deterministic non-uniform inputs and independent result buffers. + /// + [GlobalSetup] + public void Setup() + { + this.scanline = new byte[this.Count]; + this.previousScanline = new byte[this.Count]; + this.currentResult = new byte[this.Count + 1]; + this.baselineResult = new byte[this.Count + 1]; + + Random random = new(12345678); + random.NextBytes(this.scanline); + random.NextBytes(this.previousScanline); + } + + /// + /// Executes the operator-driven map/reduce traversal. + /// + /// The filter variance sum. + [Benchmark] + public int Current() + => this.Filter switch + { + PngFilterMethod.Sub => this.EncodeSubCurrent(), + PngFilterMethod.Up => this.EncodeUpCurrent(), + PngFilterMethod.Average => this.EncodeAverageCurrent(), + PngFilterMethod.Paeth => this.EncodePaethCurrent(), + _ => throw new InvalidOperationException() + }; + + /// + /// Executes the filter-specific traversal being replaced. + /// + /// The filter variance sum. + [Benchmark(Baseline = true)] + public int Baseline() + { + int sum; + + switch (this.Filter) + { + case PngFilterMethod.Sub: + PngFilterEncodeBaseline.EncodeSub( + this.scanline, + this.baselineResult, + BytesPerPixel, + out sum); + + break; + + case PngFilterMethod.Up: + PngFilterEncodeBaseline.EncodeUp( + this.scanline, + this.previousScanline, + this.baselineResult, + out sum); + + break; + + case PngFilterMethod.Average: + PngFilterEncodeBaseline.EncodeAverage( + this.scanline, + this.previousScanline, + this.baselineResult, + BytesPerPixel, + out sum); + + break; + + case PngFilterMethod.Paeth: + PngFilterEncodeBaseline.EncodePaeth( + this.scanline, + this.previousScanline, + this.baselineResult, + BytesPerPixel, + out sum); + + break; + + default: + throw new InvalidOperationException(); + } + + return sum; + } + + /// + /// Executes the current Sub encoder. + /// + private int EncodeSubCurrent() + { + SubFilter.Encode(this.scanline, this.currentResult, BytesPerPixel, out int sum); + return sum; + } + + /// + /// Executes the current Up encoder. + /// + private int EncodeUpCurrent() + { + UpFilter.Encode(this.scanline, this.previousScanline, this.currentResult, out int sum); + return sum; + } + + /// + /// Executes the current Average encoder. + /// + private int EncodeAverageCurrent() + { + AverageFilter.Encode(this.scanline, this.previousScanline, this.currentResult, BytesPerPixel, out int sum); + return sum; + } + + /// + /// Executes the current Paeth encoder. + /// + private int EncodePaethCurrent() + { + PaethFilter.Encode(this.scanline, this.previousScanline, this.currentResult, BytesPerPixel, out int sum); + return sum; + } +} diff --git a/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeAssembly.cs b/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeAssembly.cs new file mode 100644 index 000000000..bf8a18c12 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeAssembly.cs @@ -0,0 +1,104 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Formats.Png.Filters; + +namespace SixLabors.ImageSharp.Benchmarks.Codecs.Png; + +/// +/// Exposes every normalized PNG filter and retained baseline for assembly comparison. +/// +[Config(typeof(Config.Analysis))] +public class PngFilterEncodeAssembly +{ + private const int BytesPerPixel = 4; + private const int Count = 180; + + private byte[] scanline; + private byte[] previousScanline; + private byte[] currentResult; + private byte[] baselineResult; + + /// + /// Creates inputs whose suffix exercises 512-, 256-, and 128-bit register widths. + /// + [GlobalSetup] + public void Setup() + { + this.scanline = new byte[Count]; + this.previousScanline = new byte[Count]; + this.currentResult = new byte[Count + 1]; + this.baselineResult = new byte[Count + 1]; + + Random random = new(12345678); + random.NextBytes(this.scanline); + random.NextBytes(this.previousScanline); + } + + /// + /// Executes the normalized Sub encoder. + /// + [Benchmark] + public void Sub() + => SubFilter.Encode(this.scanline, this.currentResult, BytesPerPixel, out _); + + /// + /// Executes the normalized Up encoder. + /// + [Benchmark] + public void Up() + => UpFilter.Encode(this.scanline, this.previousScanline, this.currentResult, out _); + + /// + /// Executes the normalized Average encoder. + /// + [Benchmark] + public void Average() + => AverageFilter.Encode(this.scanline, this.previousScanline, this.currentResult, BytesPerPixel, out _); + + /// + /// Executes the normalized Paeth encoder. + /// + [Benchmark] + public void Paeth() + => PaethFilter.Encode(this.scanline, this.previousScanline, this.currentResult, BytesPerPixel, out _); + + /// + /// Executes the retained Sub encoder. + /// + [Benchmark] + public void BaselineSub() + => PngFilterEncodeBaseline.EncodeSub(this.scanline, this.baselineResult, BytesPerPixel, out _); + + /// + /// Executes the retained Up encoder. + /// + [Benchmark] + public void BaselineUp() + => PngFilterEncodeBaseline.EncodeUp(this.scanline, this.previousScanline, this.baselineResult, out _); + + /// + /// Executes the retained Average encoder. + /// + [Benchmark] + public void BaselineAverage() + => PngFilterEncodeBaseline.EncodeAverage( + this.scanline, + this.previousScanline, + this.baselineResult, + BytesPerPixel, + out _); + + /// + /// Executes the retained Paeth encoder. + /// + [Benchmark] + public void BaselinePaeth() + => PngFilterEncodeBaseline.EncodePaeth( + this.scanline, + this.previousScanline, + this.baselineResult, + BytesPerPixel, + out _); +} diff --git a/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeBaseline.cs b/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeBaseline.cs new file mode 100644 index 000000000..e28cc8105 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeBaseline.cs @@ -0,0 +1,470 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace SixLabors.ImageSharp.Benchmarks.Codecs.Png; + +/// +/// Retains the filter-specific PNG encode traversals for direct performance comparison. +/// +internal static class PngFilterEncodeBaseline +{ + /// + /// Executes the filter-specific Sub traversal. + /// + public static void EncodeSub( + ReadOnlySpan scanline, + Span result, + int bytesPerPixel, + out int sum) + { + ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); + ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); + sum = 0; + resultBaseRef = 1; + + nuint x = 0; + + for (; x < (uint)bytesPerPixel;) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = scan; + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + + if (Avx2.IsSupported) + { + Vector256 zero = Vector256.Zero; + Vector256 accumulator = Vector256.Zero; + + for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= scanline.Length - Vector256.Count; xLeft += (uint)Vector256.Count) + { + Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector256 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); + Vector256 residual = Avx2.Subtract(scan, left); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector256.Count; + accumulator = Avx2.Add( + accumulator, + Avx2.SumAbsoluteDifferences(Avx2.Abs(residual.AsSByte()), zero).AsInt32()); + } + + sum += Numerics.EvenReduceSum(accumulator); + } + else if (Vector.IsHardwareAccelerated) + { + Vector accumulator = Vector.Zero; + + for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= scanline.Length - Vector.Count; xLeft += (uint)Vector.Count) + { + Vector scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); + Vector residual = scan - left; + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector.Count; + Numerics.Accumulate( + ref accumulator, + Vector.AsVectorByte(Vector.Abs(Vector.AsVectorSByte(residual)))); + } + + for (int i = 0; i < Vector.Count; i++) + { + sum += (int)accumulator[i]; + } + } + + for (nuint xLeft = x - (uint)bytesPerPixel; x < (uint)scanline.Length; xLeft++) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + byte left = Unsafe.Add(ref scanBaseRef, xLeft); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = (byte)(scan - left); + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + } + + /// + /// Executes the filter-specific Up traversal. + /// + public static void EncodeUp( + ReadOnlySpan scanline, + ReadOnlySpan previousScanline, + Span result, + out int sum) + { + ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); + ref byte previousBaseRef = ref MemoryMarshal.GetReference(previousScanline); + ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); + sum = 0; + resultBaseRef = 2; + + nuint x = 0; + + if (Avx2.IsSupported) + { + Vector256 zero = Vector256.Zero; + Vector256 accumulator = Vector256.Zero; + + for (; (int)x <= scanline.Length - Vector256.Count;) + { + Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector256 above = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)); + Vector256 residual = Avx2.Subtract(scan, above); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector256.Count; + accumulator = Avx2.Add( + accumulator, + Avx2.SumAbsoluteDifferences(Avx2.Abs(residual.AsSByte()), zero).AsInt32()); + } + + sum += Numerics.EvenReduceSum(accumulator); + } + else if (Vector.IsHardwareAccelerated) + { + Vector accumulator = Vector.Zero; + + for (; (int)x <= scanline.Length - Vector.Count;) + { + Vector scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector above = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)); + Vector residual = scan - above; + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector.Count; + Numerics.Accumulate( + ref accumulator, + Vector.AsVectorByte(Vector.Abs(Vector.AsVectorSByte(residual)))); + } + + for (int i = 0; i < Vector.Count; i++) + { + sum += (int)accumulator[i]; + } + } + + for (; x < (uint)scanline.Length;) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + byte above = Unsafe.Add(ref previousBaseRef, x); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = (byte)(scan - above); + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + } + + /// + /// Executes the filter-specific Average traversal. + /// + public static void EncodeAverage( + ReadOnlySpan scanline, + ReadOnlySpan previousScanline, + Span result, + uint bytesPerPixel, + out int sum) + { + ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); + ref byte previousBaseRef = ref MemoryMarshal.GetReference(previousScanline); + ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); + sum = 0; + resultBaseRef = 3; + + nuint x = 0; + + for (; x < bytesPerPixel;) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + byte above = Unsafe.Add(ref previousBaseRef, x); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = (byte)(scan - (above >> 1)); + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + + if (Avx2.IsSupported) + { + Vector256 zero = Vector256.Zero; + Vector256 accumulator = Vector256.Zero; + Vector256 allBitsSet = Avx2.CompareEqual(accumulator, accumulator).AsByte(); + + for (nuint xLeft = x - bytesPerPixel; (int)x <= scanline.Length - Vector256.Count; xLeft += (uint)Vector256.Count) + { + Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector256 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); + Vector256 above = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)); + Vector256 average = Avx2.Xor( + Avx2.Average(Avx2.Xor(left, allBitsSet), Avx2.Xor(above, allBitsSet)), + allBitsSet); + + Vector256 residual = Avx2.Subtract(scan, average); + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector256.Count; + accumulator = Avx2.Add( + accumulator, + Avx2.SumAbsoluteDifferences(Avx2.Abs(residual.AsSByte()), zero).AsInt32()); + } + + sum += Numerics.EvenReduceSum(accumulator); + } + else if (Sse2.IsSupported) + { + Vector128 zero = Vector128.Zero; + Vector128 accumulator = Vector128.Zero; + Vector128 allBitsSet = Sse2.CompareEqual(accumulator, accumulator).AsByte(); + + for (nuint xLeft = x - bytesPerPixel; (int)x <= scanline.Length - Vector128.Count; xLeft += (uint)Vector128.Count) + { + Vector128 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector128 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); + Vector128 above = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)); + Vector128 average = Sse2.Xor( + Sse2.Average(Sse2.Xor(left, allBitsSet), Sse2.Xor(above, allBitsSet)), + allBitsSet); + + Vector128 residual = Sse2.Subtract(scan, average); + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector128.Count; + + Vector128 absolute; + + if (Ssse3.IsSupported) + { + absolute = Ssse3.Abs(residual.AsSByte()); + } + else + { + Vector128 mask = Sse2.CompareGreaterThan(zero.AsSByte(), residual.AsSByte()); + absolute = Sse2.Xor(Sse2.Add(residual.AsSByte(), mask), mask).AsByte(); + } + + accumulator = Sse2.Add( + accumulator, + Sse2.SumAbsoluteDifferences(absolute, zero).AsInt32()); + } + + sum += Numerics.EvenReduceSum(accumulator); + } + + for (nuint xLeft = x - bytesPerPixel; x < (uint)scanline.Length; xLeft++) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + byte left = Unsafe.Add(ref scanBaseRef, xLeft); + byte above = Unsafe.Add(ref previousBaseRef, x); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = (byte)(scan - ((left + above) >> 1)); + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + } + + /// + /// Executes the filter-specific Paeth traversal. + /// + public static void EncodePaeth( + ReadOnlySpan scanline, + ReadOnlySpan previousScanline, + Span result, + int bytesPerPixel, + out int sum) + { + ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); + ref byte previousBaseRef = ref MemoryMarshal.GetReference(previousScanline); + ref byte resultBaseRef = ref MemoryMarshal.GetReference(result); + sum = 0; + resultBaseRef = 4; + + nuint x = 0; + + for (; x < (uint)bytesPerPixel;) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + byte above = Unsafe.Add(ref previousBaseRef, x); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = (byte)(scan - above); + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + + if (Avx2.IsSupported) + { + Vector256 zero = Vector256.Zero; + Vector256 accumulator = Vector256.Zero; + + for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= scanline.Length - Vector256.Count; xLeft += (uint)Vector256.Count) + { + Vector256 scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector256 left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); + Vector256 above = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)); + Vector256 upperLeft = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, xLeft)); + Vector256 residual = Avx2.Subtract(scan, PaethPredictor(left, above, upperLeft)); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector256.Count; + accumulator = Avx2.Add( + accumulator, + Avx2.SumAbsoluteDifferences(Avx2.Abs(residual.AsSByte()), zero).AsInt32()); + } + + sum += Numerics.EvenReduceSum(accumulator); + } + else if (Vector.IsHardwareAccelerated) + { + Vector accumulator = Vector.Zero; + + for (nuint xLeft = x - (uint)bytesPerPixel; (int)x <= scanline.Length - Vector.Count; xLeft += (uint)Vector.Count) + { + Vector scan = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, x)); + Vector left = Unsafe.As>(ref Unsafe.Add(ref scanBaseRef, xLeft)); + Vector above = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, x)); + Vector upperLeft = Unsafe.As>(ref Unsafe.Add(ref previousBaseRef, xLeft)); + Vector residual = scan - PaethPredictor(left, above, upperLeft); + + Unsafe.As>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = residual; + x += (uint)Vector.Count; + Numerics.Accumulate( + ref accumulator, + Vector.AsVectorByte(Vector.Abs(Vector.AsVectorSByte(residual)))); + } + + for (int i = 0; i < Vector.Count; i++) + { + sum += (int)accumulator[i]; + } + } + + for (nuint xLeft = x - (uint)bytesPerPixel; x < (uint)scanline.Length; xLeft++) + { + byte scan = Unsafe.Add(ref scanBaseRef, x); + byte left = Unsafe.Add(ref scanBaseRef, xLeft); + byte above = Unsafe.Add(ref previousBaseRef, x); + byte upperLeft = Unsafe.Add(ref previousBaseRef, xLeft); + x++; + ref byte residual = ref Unsafe.Add(ref resultBaseRef, x); + residual = (byte)(scan - PaethPredictor(left, above, upperLeft)); + sum += Numerics.Abs(unchecked((sbyte)residual)); + } + } + + /// + /// Selects the scalar Paeth predictor. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static byte PaethPredictor(byte left, byte above, byte upperLeft) + { + int p = left + above - upperLeft; + int distanceLeft = Numerics.Abs(p - left); + int distanceAbove = Numerics.Abs(p - above); + int distanceUpperLeft = Numerics.Abs(p - upperLeft); + + if (distanceLeft <= distanceAbove && distanceLeft <= distanceUpperLeft) + { + return left; + } + + return distanceAbove <= distanceUpperLeft ? above : upperLeft; + } + + /// + /// Selects the AVX2 Paeth predictor. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PaethPredictor( + Vector256 left, + Vector256 above, + Vector256 upperLeft) + { + Vector256 zero = Vector256.Zero; + Vector256 aboveMinusUpper = Avx2.SubtractSaturate(above, upperLeft); + Vector256 leftMinusUpper = Avx2.SubtractSaturate(left, upperLeft); + Vector256 distanceLeft = + Avx2.Or(Avx2.SubtractSaturate(upperLeft, above), aboveMinusUpper); + + Vector256 distanceAbove = + Avx2.Or(Avx2.SubtractSaturate(upperLeft, left), leftMinusUpper); + + Vector256 sameDirection = Avx2.CompareEqual( + Avx2.CompareEqual(aboveMinusUpper, zero), + Avx2.CompareEqual(leftMinusUpper, zero)); + + Vector256 distanceUpper = Avx2.Or( + sameDirection, + Avx2.Or( + Avx2.SubtractSaturate(distanceAbove, distanceLeft), + Avx2.SubtractSaturate(distanceLeft, distanceAbove))); + + Vector256 minimumAboveUpper = Avx2.Min(distanceUpper, distanceAbove); + Vector256 aboveOrUpper = Avx2.BlendVariable( + upperLeft, + above, + Avx2.CompareEqual(minimumAboveUpper, distanceAbove)); + + return Avx2.BlendVariable( + aboveOrUpper, + left, + Avx2.CompareEqual(Avx2.Min(minimumAboveUpper, distanceLeft), distanceLeft)); + } + + /// + /// Selects the portable vector Paeth predictor. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector PaethPredictor( + Vector left, + Vector above, + Vector upperLeft) + { + Vector.Widen(left, out Vector leftLow, out Vector leftHigh); + Vector.Widen(above, out Vector aboveLow, out Vector aboveHigh); + Vector.Widen(upperLeft, out Vector upperLow, out Vector upperHigh); + + Vector lower = PaethPredictor( + Vector.AsVectorInt16(leftLow), + Vector.AsVectorInt16(aboveLow), + Vector.AsVectorInt16(upperLow)); + + Vector upper = PaethPredictor( + Vector.AsVectorInt16(leftHigh), + Vector.AsVectorInt16(aboveHigh), + Vector.AsVectorInt16(upperHigh)); + + return Vector.AsVectorByte(Vector.Narrow(lower, upper)); + } + + /// + /// Selects the portable widened Paeth predictor. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector PaethPredictor( + Vector left, + Vector above, + Vector upperLeft) + { + Vector p = left + above - upperLeft; + Vector distanceLeft = Vector.Abs(p - left); + Vector distanceAbove = Vector.Abs(p - above); + Vector distanceUpper = Vector.Abs(p - upperLeft); + + Vector chooseLeft = Vector.BitwiseAnd( + Vector.LessThanOrEqual(distanceLeft, distanceAbove), + Vector.LessThanOrEqual(distanceLeft, distanceUpper)); + + return Vector.ConditionalSelect( + chooseLeft, + left, + Vector.ConditionalSelect( + Vector.LessThanOrEqual(distanceAbove, distanceUpper), + above, + upperLeft)); + } +} diff --git a/tests/ImageSharp.Tests/Formats/Png/PngEncoderFilterTests.cs b/tests/ImageSharp.Tests/Formats/Png/PngEncoderFilterTests.cs index 3a31b395f..0e76f4fe9 100644 --- a/tests/ImageSharp.Tests/Formats/Png/PngEncoderFilterTests.cs +++ b/tests/ImageSharp.Tests/Formats/Png/PngEncoderFilterTests.cs @@ -208,6 +208,140 @@ public class PngEncoderFilterTests : MeasureFixture HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX2); } + /// + /// Verifies every encoder across pixel strides, register boundaries, and hardware widths. + /// + [Fact] + public void EncodeMatchesReferencesAcrossRegisterBoundaries() + => FeatureTestRunner.RunWithHwIntrinsicsFeature( + AssertEncodersMatchReferences, + HwIntrinsics.AllowAll + | HwIntrinsics.DisableAVX512F + | HwIntrinsics.DisableAVX2 + | HwIntrinsics.DisableHWIntrinsic); + + /// + /// Compares every filter with its independent scalar reference across SIMD boundaries and pixel strides. + /// + private static void AssertEncodersMatchReferences() + { + int[] bytesPerPixels = [1, 2, 3, 4, 6, 8]; + int[] lengths = [8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 131, 132, 133, 135, 136, 137, 257]; + + foreach (int bytesPerPixel in bytesPerPixels) + { + foreach (int length in lengths) + { + if (length < bytesPerPixel) + { + continue; + } + + byte[] scanline = new byte[length]; + byte[] previousScanline = new byte[length]; + Random random = new((bytesPerPixel * 397) + length); + random.NextBytes(scanline); + random.NextBytes(previousScanline); + + AssertFilterMatchesReference( + PngFilterMethod.Sub, + scanline, + previousScanline, + bytesPerPixel); + + AssertFilterMatchesReference( + PngFilterMethod.Up, + scanline, + previousScanline, + bytesPerPixel); + + AssertFilterMatchesReference( + PngFilterMethod.Average, + scanline, + previousScanline, + bytesPerPixel); + + AssertFilterMatchesReference( + PngFilterMethod.Paeth, + scanline, + previousScanline, + bytesPerPixel); + } + } + } + + /// + /// Compares one filter result and variance sum with its scalar reference. + /// + /// The filter to evaluate. + /// The current scanline. + /// The preceding scanline. + /// The component distance between adjacent pixels. + private static void AssertFilterMatchesReference( + PngFilterMethod filter, + byte[] scanline, + byte[] previousScanline, + int bytesPerPixel) + { + byte[] expected = new byte[scanline.Length + 1]; + byte[] actual = new byte[scanline.Length + 1]; + int expectedSum; + int actualSum; + + switch (filter) + { + case PngFilterMethod.Sub: + ReferenceImplementations.EncodeSubFilter(scanline, expected, bytesPerPixel, out expectedSum); + SubFilter.Encode(scanline, actual, bytesPerPixel, out actualSum); + break; + + case PngFilterMethod.Up: + ReferenceImplementations.EncodeUpFilter(scanline, previousScanline, expected, out expectedSum); + UpFilter.Encode(scanline, previousScanline, actual, out actualSum); + break; + + case PngFilterMethod.Average: + ReferenceImplementations.EncodeAverageFilter( + scanline, + previousScanline, + expected, + bytesPerPixel, + out expectedSum); + + AverageFilter.Encode( + scanline, + previousScanline, + actual, + (uint)bytesPerPixel, + out actualSum); + + break; + + case PngFilterMethod.Paeth: + ReferenceImplementations.EncodePaethFilter( + scanline, + previousScanline, + expected, + bytesPerPixel, + out expectedSum); + + PaethFilter.Encode( + scanline, + previousScanline, + actual, + bytesPerPixel, + out actualSum); + + break; + + default: + throw new InvalidOperationException(); + } + + Assert.Equal(expectedSum, actualSum); + Assert.Equal(expected, actual); + } + public class TestData { private readonly PngFilterMethod filter;