From c1dc1aedb4e47bb53925952a27a7957f8a5cdc4f Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sat, 25 Jul 2026 23:40:00 +1000 Subject: [PATCH] Use tensor negation for sharpen kernels --- .../Common/Helpers/TensorPrimitives.cs | 2 +- .../Helpers/TensorPrimitives_.Negate.cs | 276 ++++++++++++++++++ .../ConvolutionProcessorHelpers.cs | 20 +- .../Common/TensorPrimitivesTests.cs | 36 +++ .../ConvolutionProcessorHelpersTest.cs | 35 +++ 5 files changed, 355 insertions(+), 14 deletions(-) create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives.cs index bcfd16bb9..4bff7882f 100644 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives.cs @@ -16,7 +16,7 @@ namespace SixLabors.ImageSharp.Common.Helpers; /// implementation when ImageSharp no longer supports target frameworks that predate it. /// #pragma warning disable SA1649 // File name should match first type name -internal static class TensorPrimitives_ +internal static partial class TensorPrimitives_ #pragma warning restore SA1649 // File name should match first type name { /// diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs new file mode 100644 index 000000000..f61e4877c --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs @@ -0,0 +1,276 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +internal static partial class TensorPrimitives_ +{ + /// + /// Defines an element-wise unary operation. + /// + /// The element type. + private interface IUnaryOperator + { + /// + /// Gets a value indicating whether the operation supports vector execution. + /// + public static abstract bool Vectorizable { get; } + + /// + /// Applies the operation to a scalar value. + /// + /// The input value. + /// The operation result. + public static abstract T Invoke(T x); + + /// + /// Applies the operation to a 128-bit vector. + /// + /// The input vector. + /// The operation result. + public static abstract Vector128 Invoke(Vector128 x); + + /// + /// Applies the operation to a 256-bit vector. + /// + /// The input vector. + /// The operation result. + public static abstract Vector256 Invoke(Vector256 x); + + /// + /// Applies the operation to a 512-bit vector. + /// + /// The input vector. + /// The operation result. + public static abstract Vector512 Invoke(Vector512 x); + } + + /// + /// Computes the element-wise negation of the values in . + /// + /// The element type. + /// The values to negate. + /// The destination for the negated values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Negate(ReadOnlySpan x, Span destination) + where T : IUnaryNegationOperators + => InvokeSpanIntoSpan>(x, destination); + + /// + /// Performs an element-wise unary operation over a span. + /// + /// The element type. + /// The operation to apply. + /// The input values. + /// The destination values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeSpanIntoSpan(ReadOnlySpan x, Span destination) + where TOperator : struct, IUnaryOperator + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T destinationRef = ref MemoryMarshal.GetReference(destination); + nuint length = (uint)x.Length; + + // The dispatch matches the other compatibility pipelines: AVX-512 is reserved for large spans because + // its setup cost is not recovered by the short image-processing buffers that dominate ImageSharp. + if (TOperator.Vectorizable + && Vector512.IsHardwareAccelerated + && Vector512.IsSupported + && length >= 512) + { + InvokeUnaryVectorized512(ref xRef, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) + { + InvokeUnaryVectorized256(ref xRef, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) + { + InvokeUnaryVectorized128(ref xRef, ref destinationRef, length); + return; + } + + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i)); + } + } + + /// + /// Applies a unary operation with 128-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeUnaryVectorized128(ref T xRef, ref T destinationRef, nuint length) + where TOperator : struct, IUnaryOperator + { + nuint vectorCount = (uint)Vector128.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + + // The final vector overlaps the preceding store when the length is not a vector multiple. Loading it + // before any stores preserves same-start in-place operation because it captures the original tail. + Vector128 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, length - vectorCount)); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a unary operation with 256-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeUnaryVectorized256(ref T xRef, ref T destinationRef, nuint length) + where TOperator : struct, IUnaryOperator + { + nuint vectorCount = (uint)Vector256.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector256 end = default; + + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, length - vectorCount)); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a unary operation with 512-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeUnaryVectorized512(ref T xRef, ref T destinationRef, nuint length) + where TOperator : struct, IUnaryOperator + { + nuint vectorCount = (uint)Vector512.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector512 end = default; + + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, length - vectorCount)); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Implements element-wise negation for scalar and SIMD inputs. + /// + /// The element type. + private readonly struct NegateOperator : IUnaryOperator + where T : IUnaryNegationOperators + { + /// + public static bool Vectorizable => true; + + /// + public static T Invoke(T x) => -x; + + /// + public static Vector128 Invoke(Vector128 x) => -x; + + /// + public static Vector256 Invoke(Vector256 x) => -x; + + /// + public static Vector512 Invoke(Vector512 x) => -x; + } +} diff --git a/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs b/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs index 4aefa0dae..52cd9dbc5 100644 --- a/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs +++ b/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using System.Diagnostics.CodeAnalysis; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.Processing.Processors.Convolution; @@ -68,19 +69,12 @@ internal static class ConvolutionProcessorHelpers // Invert the kernel for sharpening. int midpointRounded = (int)midpoint; - for (int i = 0; i < size; i++) - { - if (i == midpointRounded) - { - // Calculate central value - kernel[i] = (2F * sum) - kernel[i]; - } - else - { - // invert value - kernel[i] = -kernel[i]; - } - } + float midpointValue = kernel[midpointRounded]; + TensorPrimitives_.Negate(kernel, kernel); + + // The sharpening kernel negates every Gaussian weight except its center. Restore that original + // center while adding twice the Gaussian sum so the complete kernel retains unit response. + kernel[midpointRounded] = (2F * sum) - midpointValue; // Normalize kernel so that the sum of all weights equals 1 for (int i = 0; i < size; i++) diff --git a/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs b/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs index 375fc000d..1fe64893b 100644 --- a/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs +++ b/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs @@ -117,6 +117,42 @@ public class TensorPrimitivesTests Assert.Equal(expected, inPlace); } + /// + /// Verifies that floating-point negation preserves the scalar operator's exact bit-level behavior. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void NegateSingleMatchesScalarFormula(int length) + { + float[] values = + { + float.NaN, + -0F, + 0F, + -1F, + 1F, + float.NegativeInfinity, + float.PositiveInfinity + }; + + float[] source = new float[length]; + float[] expected = new float[length]; + + for (int i = 0; i < source.Length; i++) + { + source[i] = values[i % values.Length]; + expected[i] = -source[i]; + } + + float[] destination = new float[length]; + TensorPrimitives_.Negate(source, destination); + AssertSingleBitsEqual(expected, destination); + + TensorPrimitives_.Negate(source, source); + AssertSingleBitsEqual(expected, source); + } + /// /// Verifies that integer clamping produces identical results for separate and in-place destinations. /// diff --git a/tests/ImageSharp.Tests/Processing/Processors/Convolution/ConvolutionProcessorHelpersTest.cs b/tests/ImageSharp.Tests/Processing/Processors/Convolution/ConvolutionProcessorHelpersTest.cs index 574c98371..38c635004 100644 --- a/tests/ImageSharp.Tests/Processing/Processors/Convolution/ConvolutionProcessorHelpersTest.cs +++ b/tests/ImageSharp.Tests/Processing/Processors/Convolution/ConvolutionProcessorHelpersTest.cs @@ -41,6 +41,41 @@ public class ConvolutionProcessorHelpersTest } } + /// + /// Verifies that Gaussian sharpening preserves the scalar kernel formula across scalar and SIMD lengths. + /// + /// The kernel radius. + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(9)] + [InlineData(32)] + [InlineData(80)] + public void VerifyGaussianSharpenKernel(int radius) + { + int kernelSize = (radius * 2) + 1; + float sigma = radius / 3F; + float[] expected = new float[kernelSize]; + float sum = 0F; + + for (int i = 0; i < kernelSize; i++) + { + float value = Numerics.Gaussian(i - radius, sigma); + expected[i] = value; + sum += value; + } + + for (int i = 0; i < kernelSize; i++) + { + expected[i] = i == radius ? (2F * sum) - expected[i] : -expected[i]; + expected[i] /= sum; + } + + float[] actual = ConvolutionProcessorHelpers.CreateGaussianSharpenKernel(kernelSize, sigma); + + Assert.Equal(expected, actual); + } + [Fact] public void VerifyNonSeparableMatrix() {