From ebe5cc64da3c95cad453fa782cddb8c373dbfe5a Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Fri, 24 Jul 2026 23:40:13 +1000 Subject: [PATCH] Add TensorPrimitives compatibility implementation --- shared-infrastructure | 2 +- .../ColorProfileConverterExtensionsIcc.cs | 73 +- src/ImageSharp/Common/Helpers/Numerics.cs | 186 +- .../Common/Helpers/TensorPrimitives.cs | 1566 +++++++++++++++++ .../Components/Encoder/ComponentProcessor.cs | 95 +- .../Formats/Png/Filters/UpFilter.cs | 126 +- src/ImageSharp/Formats/Webp/AlphaDecoder.cs | 27 +- .../Formats/Webp/Lossless/Vp8LHistogram.cs | 47 +- .../General/BasicMath/AddSpan.cs | 65 + .../General/BasicMath/NormalizeSpan.cs | 61 + .../TensorPrimitivesAssemblyComparison.cs | 813 +++++++++ .../ImageSharp.Benchmarks.csproj | 1 + .../Common/TensorPrimitivesTests.cs | 356 ++++ 13 files changed, 2886 insertions(+), 532 deletions(-) create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives.cs create mode 100644 tests/ImageSharp.Benchmarks/General/BasicMath/AddSpan.cs create mode 100644 tests/ImageSharp.Benchmarks/General/BasicMath/NormalizeSpan.cs create mode 100644 tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssemblyComparison.cs create mode 100644 tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs diff --git a/shared-infrastructure b/shared-infrastructure index 7ac570345..74b7f32b8 160000 --- a/shared-infrastructure +++ b/shared-infrastructure @@ -1 +1 @@ -Subproject commit 7ac5703452348d9295db31fc0912c2bd9e419dc9 +Subproject commit 74b7f32b8e41fdf8fe2f3eda54fd5a82ebbedfbc diff --git a/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs b/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs index fd99fb446..78f88932f 100644 --- a/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs +++ b/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs @@ -5,9 +5,11 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using SixLabors.ImageSharp.ColorProfiles.Conversion.Icc; using SixLabors.ImageSharp.ColorProfiles.Icc; +using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata.Profiles.Icc; @@ -658,38 +660,8 @@ internal static class ColorProfileConverterExtensionsIcc private static void ClipNegative(Span source) { - if (Vector.IsHardwareAccelerated && Vector.IsSupported && Vector.Count >= source.Length * 4) - { - // SIMD loop - int i = 0; - int simdBatchSize = Vector.Count / 4; // Number of Vector4 elements per SIMD batch - for (; i <= source.Length - simdBatchSize; i += simdBatchSize) - { - // Load the vector from source span - Vector v = Unsafe.ReadUnaligned>(ref Unsafe.As(ref source[i])); - - v = Vector.Max(v, Vector.Zero); - - // Write the vector to the destination span - Unsafe.WriteUnaligned(ref Unsafe.As(ref source[i]), v); - } - - // Scalar fallback for remaining elements - for (; i < source.Length; i++) - { - ref Vector4 s = ref source[i]; - s = Vector4.Max(s, Vector4.Zero); - } - } - else - { - // Scalar fallback if SIMD is not supported - for (int i = 0; i < source.Length; i++) - { - ref Vector4 s = ref source[i]; - s = Vector4.Max(s, Vector4.Zero); - } - } + Span values = MemoryMarshal.Cast(source); + TensorPrimitives_.Max(values, 0F, values); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -708,39 +680,10 @@ internal static class ColorProfileConverterExtensionsIcc private static void LabToLab(Span source, Span destination, [ConstantExpected] float scale) { - if (Vector.IsHardwareAccelerated && Vector.IsSupported) - { - Vector vScale = new(scale); - int i = 0; - - // SIMD loop - int simdBatchSize = Vector.Count / 4; // Number of Vector4 elements per SIMD batch - for (; i <= source.Length - simdBatchSize; i += simdBatchSize) - { - // Load the vector from source span - Vector v = Unsafe.ReadUnaligned>(ref Unsafe.As(ref source[i])); - - // Scale the vector - v *= vScale; - - // Write the scaled vector to the destination span - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination[i]), v); - } - - // Scalar fallback for remaining elements - for (; i < source.Length; i++) - { - destination[i] = source[i] * scale; - } - } - else - { - // Scalar fallback if SIMD is not supported - for (int i = 0; i < source.Length; i++) - { - destination[i] = source[i] * scale; - } - } + TensorPrimitives_.Multiply( + MemoryMarshal.Cast(source), + scale, + MemoryMarshal.Cast(destination)); } private class ConversionParams diff --git a/src/ImageSharp/Common/Helpers/Numerics.cs b/src/ImageSharp/Common/Helpers/Numerics.cs index 5ffed8eb7..8980d2b53 100644 --- a/src/ImageSharp/Common/Helpers/Numerics.cs +++ b/src/ImageSharp/Common/Helpers/Numerics.cs @@ -329,22 +329,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, byte min, byte max) - { - Span remainder = span[ClampReduce(span, min, max)..]; - - if (remainder.Length > 0) - { - ref byte remainderStart = ref MemoryMarshal.GetReference(remainder); - ref byte remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length); - - while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd)) - { - remainderStart = Clamp(remainderStart, min, max); - - remainderStart = ref Unsafe.Add(ref remainderStart, 1); - } - } - } + => TensorPrimitives_.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -354,22 +339,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, uint min, uint max) - { - Span remainder = span[ClampReduce(span, min, max)..]; - - if (remainder.Length > 0) - { - ref uint remainderStart = ref MemoryMarshal.GetReference(remainder); - ref uint remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length); - - while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd)) - { - remainderStart = Clamp(remainderStart, min, max); - - remainderStart = ref Unsafe.Add(ref remainderStart, 1); - } - } - } + => TensorPrimitives_.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -379,22 +349,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, int min, int max) - { - Span remainder = span[ClampReduce(span, min, max)..]; - - if (remainder.Length > 0) - { - ref int remainderStart = ref MemoryMarshal.GetReference(remainder); - ref int remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length); - - while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd)) - { - remainderStart = Clamp(remainderStart, min, max); - - remainderStart = ref Unsafe.Add(ref remainderStart, 1); - } - } - } + => TensorPrimitives_.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -404,22 +359,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, float min, float max) - { - Span remainder = span[ClampReduce(span, min, max)..]; - - if (remainder.Length > 0) - { - ref float remainderStart = ref MemoryMarshal.GetReference(remainder); - ref float remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length); - - while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd)) - { - remainderStart = Clamp(remainderStart, min, max); - - remainderStart = ref Unsafe.Add(ref remainderStart, 1); - } - } - } + => TensorPrimitives_.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -429,87 +369,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, double min, double max) - { - Span remainder = span[ClampReduce(span, min, max)..]; - - if (remainder.Length > 0) - { - ref double remainderStart = ref MemoryMarshal.GetReference(remainder); - ref double remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length); - - while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd)) - { - remainderStart = Clamp(remainderStart, min, max); - - remainderStart = ref Unsafe.Add(ref remainderStart, 1); - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int ClampReduce(Span span, T min, T max) - where T : unmanaged - { - if (Vector.IsHardwareAccelerated && span.Length >= Vector.Count) - { - int remainder = ModuloP2(span.Length, Vector.Count); - int adjustedCount = span.Length - remainder; - - if (adjustedCount > 0) - { - ClampImpl(span[..adjustedCount], min, max); - } - - return adjustedCount; - } - - return 0; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ClampImpl(Span span, T min, T max) - where T : unmanaged - { - ref T sRef = ref MemoryMarshal.GetReference(span); - Vector vmin = new(min); - Vector vmax = new(max); - - nint n = (nint)(uint)span.Length / Vector.Count; - nint m = Modulo4(n); - nint u = n - m; - - ref Vector vs0 = ref Unsafe.As>(ref MemoryMarshal.GetReference(span)); - ref Vector vs1 = ref Unsafe.Add(ref vs0, 1); - ref Vector vs2 = ref Unsafe.Add(ref vs0, 2); - ref Vector vs3 = ref Unsafe.Add(ref vs0, 3); - ref Vector vsEnd = ref Unsafe.Add(ref vs0, u); - - while (Unsafe.IsAddressLessThan(ref vs0, ref vsEnd)) - { - vs0 = Vector.Min(Vector.Max(vmin, vs0), vmax); - vs1 = Vector.Min(Vector.Max(vmin, vs1), vmax); - vs2 = Vector.Min(Vector.Max(vmin, vs2), vmax); - vs3 = Vector.Min(Vector.Max(vmin, vs3), vmax); - - vs0 = ref Unsafe.Add(ref vs0, 4); - vs1 = ref Unsafe.Add(ref vs1, 4); - vs2 = ref Unsafe.Add(ref vs2, 4); - vs3 = ref Unsafe.Add(ref vs3, 4); - } - - if (m > 0) - { - vs0 = ref vsEnd; - vsEnd = ref Unsafe.Add(ref vsEnd, m); - - while (Unsafe.IsAddressLessThan(ref vs0, ref vsEnd)) - { - vs0 = Vector.Min(Vector.Max(vmin, vs0), vmax); - - vs0 = ref Unsafe.Add(ref vs0, 1); - } - } - } + => TensorPrimitives_.Clamp(span, min, max, span); /// /// Pre-multiplies the "x", "y", "z" components of a vector by its "w" component leaving the "w" component intact. @@ -1210,39 +1070,5 @@ internal static class Numerics /// The sum of the values in . [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Normalize(Span span, float sum) - { - if (Vector256.IsHardwareAccelerated) - { - ref float startRef = ref MemoryMarshal.GetReference(span); - ref float endRef = ref Unsafe.Add(ref startRef, span.Length & ~7); - Vector256 sum256 = Vector256.Create(sum); - - while (Unsafe.IsAddressLessThan(ref startRef, ref endRef)) - { - Unsafe.As>(ref startRef) /= sum256; - startRef = ref Unsafe.Add(ref startRef, (nuint)8); - } - - if ((span.Length & 7) >= 4) - { - Unsafe.As>(ref startRef) /= sum256.GetLower(); - startRef = ref Unsafe.Add(ref startRef, (nuint)4); - } - - endRef = ref Unsafe.Add(ref startRef, span.Length & 3); - - while (Unsafe.IsAddressLessThan(ref startRef, ref endRef)) - { - startRef /= sum; - startRef = ref Unsafe.Add(ref startRef, (nuint)1); - } - } - else - { - for (int i = 0; i < span.Length; i++) - { - span[i] /= sum; - } - } - } + => TensorPrimitives_.Divide(span, sum, span); } diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives.cs new file mode 100644 index 000000000..7f97776ab --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives.cs @@ -0,0 +1,1566 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +/// +/// Provides compatibility implementations for tensor operations that are not available on every target framework. +/// +/// +/// The API shape follows System.Numerics.Tensors.TensorPrimitives so call sites can move to the runtime +/// implementation when ImageSharp no longer supports target frameworks that predate it. +/// +#pragma warning disable SA1649 // File name should match first type name +internal static class TensorPrimitives_ +#pragma warning restore SA1649 // File name should match first type name +{ + /// + /// Defines an element-wise binary operation. + /// + /// The element type. + private interface IBinaryOperator + { + /// + /// Gets a value indicating whether the operation supports vector execution. + /// + public static abstract bool Vectorizable { get; } + + /// + /// Applies the operation to scalar values. + /// + /// The first value. + /// The second value. + /// The operation result. + public static abstract T Invoke(T x, T y); + + /// + /// Applies the operation to 128-bit vectors. + /// + /// The first vector. + /// The second vector. + /// The operation result. + public static abstract Vector128 Invoke(Vector128 x, Vector128 y); + + /// + /// Applies the operation to 256-bit vectors. + /// + /// The first vector. + /// The second vector. + /// The operation result. + public static abstract Vector256 Invoke(Vector256 x, Vector256 y); + + /// + /// Applies the operation to 512-bit vectors. + /// + /// The first vector. + /// The second vector. + /// The operation result. + public static abstract Vector512 Invoke(Vector512 x, Vector512 y); + } + + /// + /// Defines an element-wise ternary operation. + /// + /// The element type. + private interface ITernaryOperator + { + /// + /// Gets a value indicating whether the operation supports vector execution. + /// + public static abstract bool Vectorizable { get; } + + /// + /// Applies the operation to scalar values. + /// + /// The first value. + /// The second value. + /// The third value. + /// The operation result. + public static abstract T Invoke(T x, T y, T z); + + /// + /// Applies the operation to 128-bit vectors. + /// + /// The first vector. + /// The second vector. + /// The third vector. + /// The operation result. + public static abstract Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z); + + /// + /// Applies the operation to 256-bit vectors. + /// + /// The first vector. + /// The second vector. + /// The third vector. + /// The operation result. + public static abstract Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z); + + /// + /// Applies the operation to 512-bit vectors. + /// + /// The first vector. + /// The second vector. + /// The third vector. + /// The operation result. + public static abstract Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z); + } + + /// + /// Computes the element-wise result of clamping to the inclusive range specified + /// by and . + /// + /// The element type. + /// The values to clamp. + /// The inclusive lower bound. + /// The inclusive upper bound. + /// The destination for the clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Clamp(ReadOnlySpan x, T min, T max, Span destination) + where T : INumber + => InvokeSpanScalarScalarIntoSpan>(x, min, max, destination); + + /// + /// Computes the element-wise sum of the values in and . + /// + /// The element type. + /// The first addends. + /// The second addends. + /// The destination for the sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IAdditionOperators, IAdditiveIdentity + => InvokeSpanSpanIntoSpan>(x, y, destination); + + /// + /// Computes the element-wise result of dividing the values in by . + /// + /// The element type. + /// The dividend values. + /// The divisor. + /// The destination for the quotient values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Divide(ReadOnlySpan x, T y, Span destination) + where T : IDivisionOperators + => InvokeSpanScalarIntoSpanForDivision>(x, y, destination); + + /// + /// Computes the element-wise maximum of the values in and . + /// + /// The element type. + /// The values to compare. + /// The value to compare with each element. + /// The destination for the maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Max(ReadOnlySpan x, T y, Span destination) + where T : INumber + => InvokeSpanScalarIntoSpan>(x, y, destination); + + /// + /// Computes the element-wise product of the values in and . + /// + /// The element type. + /// The multiplicands. + /// The multiplier. + /// The destination for the products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Multiply(ReadOnlySpan x, T y, Span destination) + where T : IMultiplyOperators, IMultiplicativeIdentity + => InvokeSpanScalarIntoSpan>(x, y, destination); + + /// + /// Performs an element-wise binary operation between two spans. + /// + /// The element type. + /// The operation to apply. + /// The first input values. + /// The second input values. + /// The destination values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeSpanSpanIntoSpan( + ReadOnlySpan x, + ReadOnlySpan y, + Span destination) + where TOperator : struct, IBinaryOperator + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + ref T destinationRef = ref MemoryMarshal.GetReference(destination); + nuint length = (uint)x.Length; + + // AVX-512 setup only pays off for larger multi-byte inputs. Byte addition remains on AVX2 because direct + // PNG/WebP measurements show that its higher lane count does not recover the wider dispatch cost. + // Each pipeline preloads its final inputs when a tail overlaps so same-start in-place operation remains correct. + if (TOperator.Vectorizable + && Vector512.IsHardwareAccelerated + && Vector512.IsSupported + && Unsafe.SizeOf() > 1 + && length >= 512) + { + InvokeVectorized512(ref xRef, ref yRef, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) + { + InvokeVectorized256(ref xRef, ref yRef, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) + { + InvokeVectorized128(ref xRef, ref yRef, ref destinationRef, length); + return; + } + + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), Unsafe.Add(ref yRef, i)); + } + } + + /// + /// Performs an element-wise binary operation between a span and a scalar. + /// + /// The element type. + /// The operation to apply. + /// The input values. + /// The scalar input. + /// The destination values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeSpanScalarIntoSpan( + ReadOnlySpan x, + T y, + Span destination) + where TOperator : struct, IBinaryOperator + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T destinationRef = ref MemoryMarshal.GetReference(destination); + nuint length = (uint)x.Length; + + // The runtime-style unrolled 512-bit pipeline wins on large inputs, but its setup cost regresses the + // shorter JPEG and ICC buffers. Measurements put the crossover safely below 512 elements. + if (TOperator.Vectorizable + && Vector512.IsHardwareAccelerated + && Vector512.IsSupported + && length >= 512) + { + InvokeVectorized512(ref xRef, y, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) + { + InvokeVectorized256(ref xRef, y, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) + { + InvokeVectorized128(ref xRef, y, ref destinationRef, length); + return; + } + + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); + } + } + + /// + /// Performs element-wise division using thresholds measured for ImageSharp normalization workloads. + /// + /// The element type. + /// The division operation to apply. + /// The input values. + /// The scalar divisor. + /// The destination values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeSpanScalarIntoSpanForDivision( + ReadOnlySpan x, + T y, + Span destination) + where TOperator : struct, IBinaryOperator + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T destinationRef = ref MemoryMarshal.GetReference(destination); + nuint length = (uint)x.Length; + + // AVX-512 only wins once there is enough work to amortize its wider dispatch and division latency. + // Eight vectors is also the runtime pipeline's unrolled-loop boundary, while shorter inputs retain + // the lower setup cost of 256-bit vectors. + if (TOperator.Vectorizable + && Vector512.IsHardwareAccelerated + && Vector512.IsSupported + && length >= (uint)(Vector512.Count * 8)) + { + InvokeVectorized512(ref xRef, y, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) + { + InvokeVectorized256(ref xRef, y, ref destinationRef, length); + return; + } + + // Four values fill one 128-bit float vector. Processing exactly one packed prefix before the scalar + // remainder avoids the overlapping second vector that regresses the common seven-element normalization. + if (TOperator.Vectorizable + && Vector128.IsHardwareAccelerated + && Vector128.IsSupported + && length >= (uint)Vector128.Count) + { + nuint vectorCount = (uint)Vector128.Count; + Vector128 yVector = Vector128.Create(y); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef), yVector).StoreUnsafe(ref destinationRef); + + for (nuint i = vectorCount; i < length; i++) + { + Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); + } + + return; + } + + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); + } + } + + /// + /// Performs an element-wise ternary operation between a span and two scalars. + /// + /// The element type. + /// The operation to apply. + /// The input values. + /// The first scalar input. + /// The second scalar input. + /// The destination values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeSpanScalarScalarIntoSpan( + ReadOnlySpan x, + T y, + T z, + Span destination) + where TOperator : struct, ITernaryOperator + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T destinationRef = ref MemoryMarshal.GetReference(destination); + nuint length = (uint)x.Length; + + // This dispatch mirrors the runtime pipeline: large inputs use the widest available registers while + // short inputs fall through to a width that fits, keeping the operator contract identical at every length. + if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512.IsSupported && length >= (uint)Vector512.Count) + { + InvokeVectorized512(ref xRef, y, z, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) + { + InvokeVectorized256(ref xRef, y, z, ref destinationRef, length); + return; + } + + if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) + { + InvokeVectorized128(ref xRef, y, z, ref destinationRef, length); + return; + } + + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y, z); + } + } + + /// + /// Applies a binary operation between two spans with 128-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first element of the first input. + /// The first element of the second input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized128( + ref T xRef, + ref T yRef, + ref T destinationRef, + nuint length) + where TOperator : struct, IBinaryOperator + { + nuint vectorCount = (uint)Vector128.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + + // When a tail exists, both final inputs are loaded before any stores. This permits either source to also + // be the destination when the tail starts inside the range written by the preceding full vector. + Vector128 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector128.LoadUnsafe(ref xRef, length - vectorCount), + Vector128.LoadUnsafe(ref yRef, length - vectorCount)); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), Vector128.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a binary operation between two spans with 256-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first element of the first input. + /// The first element of the second input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized256( + ref T xRef, + ref T yRef, + ref T destinationRef, + nuint length) + where TOperator : struct, IBinaryOperator + { + nuint vectorCount = (uint)Vector256.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector256 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector256.LoadUnsafe(ref xRef, length - vectorCount), + Vector256.LoadUnsafe(ref yRef, length - vectorCount)); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), Vector256.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a binary operation between two spans with 512-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first element of the first input. + /// The first element of the second input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized512( + ref T xRef, + ref T yRef, + ref T destinationRef, + nuint length) + where TOperator : struct, IBinaryOperator + { + nuint vectorCount = (uint)Vector512.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector512 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector512.LoadUnsafe(ref xRef, length - vectorCount), + Vector512.LoadUnsafe(ref yRef, length - vectorCount)); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), Vector512.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a binary operation between a span and a scalar with 128-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The scalar input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized128( + ref T xRef, + T y, + ref T destinationRef, + nuint length) + where TOperator : struct, IBinaryOperator + { + nuint vectorCount = (uint)Vector128.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector128 yVector = Vector128.Create(y); + + // When a tail exists, preloading its final vector is required for in-place operation because it must + // observe the original values before an earlier overlapping store writes them. + Vector128 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector128.LoadUnsafe(ref xRef, length - vectorCount), + yVector); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a binary operation with 256-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The scalar input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized256( + ref T xRef, + T y, + ref T destinationRef, + nuint length) + where TOperator : struct, IBinaryOperator + { + nuint vectorCount = (uint)Vector256.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector256 yVector = Vector256.Create(y); + Vector256 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector256.LoadUnsafe(ref xRef, length - vectorCount), + yVector); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a binary operation with 512-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The scalar input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized512( + ref T xRef, + T y, + ref T destinationRef, + nuint length) + where TOperator : struct, IBinaryOperator + { + nuint vectorCount = (uint)Vector512.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector512 yVector = Vector512.Create(y); + Vector512 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector512.LoadUnsafe(ref xRef, length - vectorCount), + yVector); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Selects maximum single-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MaxSingle(Vector128 x, Vector128 y) + { + // The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a + // first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order. + Vector128 result = Vector128.Max(x, y); + result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); + + return Vector128.ConditionalSelect( + Vector128.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum single-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MaxSingle(Vector256 x, Vector256 y) + { + Vector256 result = Vector256.Max(x, y); + result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); + + return Vector256.ConditionalSelect( + Vector256.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum single-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 MaxSingle(Vector512 x, Vector512 y) + { + Vector512 result = Vector512.Max(x, y); + result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); + + return Vector512.ConditionalSelect( + Vector512.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum double-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MaxDouble(Vector128 x, Vector128 y) + { + Vector128 result = Vector128.Max(x, y); + result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); + + return Vector128.ConditionalSelect( + Vector128.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum double-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MaxDouble(Vector256 x, Vector256 y) + { + Vector256 result = Vector256.Max(x, y); + result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); + + return Vector256.ConditionalSelect( + Vector256.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum double-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 MaxDouble(Vector512 x, Vector512 y) + { + Vector512 result = Vector512.Max(x, y); + result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); + + return Vector512.ConditionalSelect( + Vector512.Equals(x, y), + x & y, + result); + } + + /// + /// Applies a ternary operation with 128-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The first scalar input. + /// The second scalar input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized128( + ref T xRef, + T y, + T z, + ref T destinationRef, + nuint length) + where TOperator : struct, ITernaryOperator + { + nuint vectorCount = (uint)Vector128.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector128 yVector = Vector128.Create(y); + Vector128 zVector = Vector128.Create(z); + Vector128 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector128.LoadUnsafe(ref xRef, length - vectorCount), + yVector, + zVector); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a ternary operation with 256-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The first scalar input. + /// The second scalar input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized256( + ref T xRef, + T y, + T z, + ref T destinationRef, + nuint length) + where TOperator : struct, ITernaryOperator + { + nuint vectorCount = (uint)Vector256.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector256 yVector = Vector256.Create(y); + Vector256 zVector = Vector256.Create(z); + Vector256 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector256.LoadUnsafe(ref xRef, length - vectorCount), + yVector, + zVector); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Applies a ternary operation with 512-bit vectors. + /// + /// The element type. + /// The operation to apply. + /// The first input element. + /// The first scalar input. + /// The second scalar input. + /// The first destination element. + /// The number of elements to process. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InvokeVectorized512( + ref T xRef, + T y, + T z, + ref T destinationRef, + nuint length) + where TOperator : struct, ITernaryOperator + { + nuint vectorCount = (uint)Vector512.Count; + nuint vectorsPerLoop = vectorCount * 8; + nuint index = 0; + Vector512 yVector = Vector512.Create(y); + Vector512 zVector = Vector512.Create(z); + Vector512 end = default; + if ((length % vectorCount) != 0) + { + end = TOperator.Invoke( + Vector512.LoadUnsafe(ref xRef, length - vectorCount), + yVector, + zVector); + } + + while ((length - index) >= vectorsPerLoop) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); + + index += vectorsPerLoop; + } + + while ((length - index) >= vectorCount) + { + TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); + index += vectorCount; + } + + if (index != length) + { + end.StoreUnsafe(ref destinationRef, length - vectorCount); + } + } + + /// + /// Clamps single-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 ClampSingle( + Vector128 value, + Vector128 min, + Vector128 max) + { + // Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the + // first operand and select negative zero when equal values have different signs. + Vector128 maximum = Vector128.ConditionalSelect( + Vector128.LessThan(min, value) + | ~Vector128.Equals(value, value) + | (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), + value, + min); + + return Vector128.ConditionalSelect( + Vector128.LessThan(maximum, max) + | ~Vector128.Equals(maximum, maximum) + | (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), + maximum, + max); + } + + /// + /// Clamps single-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 ClampSingle( + Vector256 value, + Vector256 min, + Vector256 max) + { + Vector256 maximum = Vector256.ConditionalSelect( + Vector256.LessThan(min, value) + | ~Vector256.Equals(value, value) + | (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), + value, + min); + + return Vector256.ConditionalSelect( + Vector256.LessThan(maximum, max) + | ~Vector256.Equals(maximum, maximum) + | (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), + maximum, + max); + } + + /// + /// Clamps single-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 ClampSingle( + Vector512 value, + Vector512 min, + Vector512 max) + { + Vector512 maximum = Vector512.ConditionalSelect( + Vector512.LessThan(min, value) + | ~Vector512.Equals(value, value) + | (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), + value, + min); + + return Vector512.ConditionalSelect( + Vector512.LessThan(maximum, max) + | ~Vector512.Equals(maximum, maximum) + | (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), + maximum, + max); + } + + /// + /// Clamps double-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 ClampDouble( + Vector128 value, + Vector128 min, + Vector128 max) + { + Vector128 maximum = Vector128.ConditionalSelect( + Vector128.LessThan(min, value) + | ~Vector128.Equals(value, value) + | (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), + value, + min); + + return Vector128.ConditionalSelect( + Vector128.LessThan(maximum, max) + | ~Vector128.Equals(maximum, maximum) + | (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), + maximum, + max); + } + + /// + /// Clamps double-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 ClampDouble( + Vector256 value, + Vector256 min, + Vector256 max) + { + Vector256 maximum = Vector256.ConditionalSelect( + Vector256.LessThan(min, value) + | ~Vector256.Equals(value, value) + | (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), + value, + min); + + return Vector256.ConditionalSelect( + Vector256.LessThan(maximum, max) + | ~Vector256.Equals(maximum, maximum) + | (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), + maximum, + max); + } + + /// + /// Clamps double-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 ClampDouble( + Vector512 value, + Vector512 min, + Vector512 max) + { + Vector512 maximum = Vector512.ConditionalSelect( + Vector512.LessThan(min, value) + | ~Vector512.Equals(value, value) + | (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), + value, + min); + + return Vector512.ConditionalSelect( + Vector512.LessThan(maximum, max) + | ~Vector512.Equals(maximum, maximum) + | (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), + maximum, + max); + } + + /// + /// Determines whether has the same vector division support as . + /// + /// The element type. + /// when is a 32-bit signed native integer type. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsInt32Like() + => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); + + /// + /// Adds corresponding values. + /// + /// The element type. + private readonly struct AddOperator : IBinaryOperator + where T : IAdditionOperators, IAdditiveIdentity + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Adds scalar values. + /// + /// The first addend. + /// The second addend. + /// The sum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => x + y; + + /// + /// Adds 128-bit vectors. + /// + /// The first addends. + /// The second addends. + /// The sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y; + + /// + /// Adds 256-bit vectors. + /// + /// The first addends. + /// The second addends. + /// The sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y; + + /// + /// Adds 512-bit vectors. + /// + /// The first addends. + /// The second addends. + /// The sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y; + } + + /// + /// Clamps values using the complete runtime tensor contract, including signed-zero correction. + /// + /// The element type. + private readonly struct ClampOperator : ITernaryOperator + where T : INumber + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Clamps a scalar value. + /// + /// The value. + /// The inclusive lower bound. + /// The inclusive upper bound. + /// The clamped value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T min, T max) + => Vector128.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max); + + /// + /// Clamps a 128-bit vector. + /// + /// The values. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 min, Vector128 max) + { + if (typeof(T) == typeof(float)) + { + Vector128 result = ClampSingle( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref min), + Unsafe.As, Vector128>(ref max)); + + return Unsafe.As, Vector128>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector128 result = ClampDouble( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref min), + Unsafe.As, Vector128>(ref max)); + + return Unsafe.As, Vector128>(ref result); + } + + return Vector128_.Clamp(x, min, max); + } + + /// + /// Clamps a 256-bit vector. + /// + /// The values. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 min, Vector256 max) + { + if (typeof(T) == typeof(float)) + { + Vector256 result = ClampSingle( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref min), + Unsafe.As, Vector256>(ref max)); + + return Unsafe.As, Vector256>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector256 result = ClampDouble( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref min), + Unsafe.As, Vector256>(ref max)); + + return Unsafe.As, Vector256>(ref result); + } + + return Vector256_.Clamp(x, min, max); + } + + /// + /// Clamps a 512-bit vector. + /// + /// The values. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 min, Vector512 max) + { + if (typeof(T) == typeof(float)) + { + Vector512 result = ClampSingle( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref min), + Unsafe.As, Vector512>(ref max)); + + return Unsafe.As, Vector512>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector512 result = ClampDouble( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref min), + Unsafe.As, Vector512>(ref max)); + + return Unsafe.As, Vector512>(ref result); + } + + return Vector512_.Clamp(x, min, max); + } + } + + /// + /// Selects the maximum corresponding values. + /// + /// The element type. + private readonly struct MaxOperator : IBinaryOperator + where T : INumber + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Selects the maximum scalar value. + /// + /// The first value. + /// The second value. + /// The maximum value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.Max(x, y); + + /// + /// Selects the maximum values from 128-bit vectors. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (typeof(T) == typeof(float)) + { + Vector128 result = MaxSingle( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref y)); + + return Unsafe.As, Vector128>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector128 result = MaxDouble( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref y)); + + return Unsafe.As, Vector128>(ref result); + } + + return Vector128.Max(x, y); + } + + /// + /// Selects the maximum values from 256-bit vectors. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float)) + { + Vector256 result = MaxSingle( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref y)); + + return Unsafe.As, Vector256>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector256 result = MaxDouble( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref y)); + + return Unsafe.As, Vector256>(ref result); + } + + return Vector256.Max(x, y); + } + + /// + /// Selects the maximum values from 512-bit vectors. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float)) + { + Vector512 result = MaxSingle( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref y)); + + return Unsafe.As, Vector512>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector512 result = MaxDouble( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref y)); + + return Unsafe.As, Vector512>(ref result); + } + + return Vector512.Max(x, y); + } + } + + /// + /// Multiplies corresponding values. + /// + /// The element type. + private readonly struct MultiplyOperator : IBinaryOperator + where T : IMultiplyOperators, IMultiplicativeIdentity + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Multiplies scalar values. + /// + /// The multiplicand. + /// The multiplier. + /// The product. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => x * y; + + /// + /// Multiplies 128-bit vectors. + /// + /// The multiplicands. + /// The multipliers. + /// The products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => x * y; + + /// + /// Multiplies 256-bit vectors. + /// + /// The multiplicands. + /// The multipliers. + /// The products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => x * y; + + /// + /// Multiplies 512-bit vectors. + /// + /// The multiplicands. + /// The multipliers. + /// The products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; + } + + /// + /// Divides values by a scalar. + /// + /// The element type. + private readonly struct DivideOperator : IBinaryOperator + where T : IDivisionOperators + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => typeof(T) == typeof(float) + || typeof(T) == typeof(double) + || (Vector256.IsHardwareAccelerated && IsInt32Like()); + + /// + /// Divides scalar values. + /// + /// The dividend. + /// The divisor. + /// The quotient. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => x / y; + + /// + /// Divides 128-bit vectors. + /// + /// The dividends. + /// The divisors. + /// The quotients. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; + + /// + /// Divides 256-bit vectors. + /// + /// The dividends. + /// The divisors. + /// The quotients. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; + + /// + /// Divides 512-bit vectors. + /// + /// The dividends. + /// The divisors. + /// The quotients. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; + } +} diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs index 1b0a17704..177e9b44f 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs @@ -5,8 +5,8 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder; @@ -116,52 +116,7 @@ internal class ComponentProcessor : IDisposable } static void SumVertical(Span target, Span source) - { - if (Avx.IsSupported) - { - ref Vector256 targetVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); - ref Vector256 sourceVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); - - // Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed - DebugGuard.IsTrue(source.Length % 8 == 0, "source must be multiple of 8"); - nuint count = source.Vector256Count(); - for (nuint i = 0; i < count; i++) - { - Unsafe.Add(ref targetVectorRef, i) = Avx.Add(Unsafe.Add(ref targetVectorRef, i), Unsafe.Add(ref sourceVectorRef, i)); - } - } - else if (AdvSimd.IsSupported) - { - ref Vector128 targetVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); - ref Vector128 sourceVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); - - // Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed - DebugGuard.IsTrue(source.Length % 8 == 0, "source must be multiple of 8"); - nuint count = source.Vector128Count(); - for (nuint i = 0; i < count; i++) - { - Unsafe.Add(ref targetVectorRef, i) = AdvSimd.Add(Unsafe.Add(ref targetVectorRef, i), Unsafe.Add(ref sourceVectorRef, i)); - } - } - else - { - ref Vector targetVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); - ref Vector sourceVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); - - nuint count = source.VectorCount(); - for (nuint i = 0; i < count; i++) - { - Unsafe.Add(ref targetVectorRef, i) += Unsafe.Add(ref sourceVectorRef, i); - } - - ref float targetRef = ref MemoryMarshal.GetReference(target); - ref float sourceRef = ref MemoryMarshal.GetReference(source); - for (nuint i = count * (uint)Vector.Count; i < (uint)source.Length; i++) - { - Unsafe.Add(ref targetRef, i) += Unsafe.Add(ref sourceRef, i); - } - } - } + => TensorPrimitives_.Add(target, source, target); static void SumHorizontal(Span target, int factor) { @@ -209,50 +164,6 @@ internal class ComponentProcessor : IDisposable } static void MultiplyToAverage(Span target, float multiplier) - { - if (Avx.IsSupported) - { - ref Vector256 targetVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); - - // Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed - DebugGuard.IsTrue(target.Length % 8 == 0, "target must be multiple of 8"); - nuint count = target.Vector256Count(); - Vector256 multiplierVector = Vector256.Create(multiplier); - for (nuint i = 0; i < count; i++) - { - Unsafe.Add(ref targetVectorRef, i) = Avx.Multiply(Unsafe.Add(ref targetVectorRef, i), multiplierVector); - } - } - else if (AdvSimd.IsSupported) - { - ref Vector128 targetVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); - - // Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed - DebugGuard.IsTrue(target.Length % 8 == 0, "target must be multiple of 8"); - nuint count = target.Vector128Count(); - Vector128 multiplierVector = Vector128.Create(multiplier); - for (nuint i = 0; i < count; i++) - { - Unsafe.Add(ref targetVectorRef, i) = AdvSimd.Multiply(Unsafe.Add(ref targetVectorRef, i), multiplierVector); - } - } - else - { - ref Vector targetVectorRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); - - nuint count = target.VectorCount(); - Vector multiplierVector = new(multiplier); - for (nuint i = 0; i < count; i++) - { - Unsafe.Add(ref targetVectorRef, i) *= multiplierVector; - } - - ref float targetRef = ref MemoryMarshal.GetReference(target); - for (nuint i = count * (uint)Vector.Count; i < (uint)target.Length; i++) - { - Unsafe.Add(ref targetRef, i) *= multiplier; - } - } - } + => TensorPrimitives_.Multiply(target, multiplier, target); } } diff --git a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs index 405d89e6c..d9c8e36d6 100644 --- a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs +++ b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs @@ -5,8 +5,8 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.Formats.Png.Filters; @@ -27,128 +27,8 @@ internal static class UpFilter { DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); - if (Avx2.IsSupported) - { - DecodeAvx2(scanline, previousScanline); - } - else if (Sse2.IsSupported) - { - DecodeSse2(scanline, previousScanline); - } - else if (AdvSimd.IsSupported) - { - DecodeArm(scanline, previousScanline); - } - else - { - DecodeScalar(scanline, previousScanline); - } - } - - private static void DecodeAvx2(Span scanline, Span previousScanline) - { - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - - // Up(x) + Prior(x) - int rb = scanline.Length; - nuint offset = 1; - while (rb >= Vector256.Count) - { - ref byte scanRef = ref Unsafe.Add(ref scanBaseRef, offset); - Vector256 prior = Unsafe.As>(ref scanRef); - Vector256 up = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, offset)); - - Unsafe.As>(ref scanRef) = Avx2.Add(up, prior); - - offset += (uint)Vector256.Count; - rb -= Vector256.Count; - } - - // Handle left over. - for (nuint i = offset; i < (uint)scanline.Length; i++) - { - ref byte scan = ref Unsafe.Add(ref scanBaseRef, offset); - byte above = Unsafe.Add(ref prevBaseRef, offset); - scan = (byte)(scan + above); - offset++; - } - } - - private static void DecodeSse2(Span scanline, Span previousScanline) - { - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - - // Up(x) + Prior(x) - int rb = scanline.Length; - nuint offset = 1; - while (rb >= Vector128.Count) - { - ref byte scanRef = ref Unsafe.Add(ref scanBaseRef, offset); - Vector128 prior = Unsafe.As>(ref scanRef); - Vector128 up = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, offset)); - - Unsafe.As>(ref scanRef) = Sse2.Add(up, prior); - - offset += (uint)Vector128.Count; - rb -= Vector128.Count; - } - - // Handle left over. - for (nuint i = offset; i < (uint)scanline.Length; i++) - { - ref byte scan = ref Unsafe.Add(ref scanBaseRef, offset); - byte above = Unsafe.Add(ref prevBaseRef, offset); - scan = (byte)(scan + above); - offset++; - } - } - - private static void DecodeArm(Span scanline, Span previousScanline) - { - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - - // Up(x) + Prior(x) - int rb = scanline.Length; - nuint offset = 1; - const int bytesPerBatch = 16; - while (rb >= bytesPerBatch) - { - ref byte scanRef = ref Unsafe.Add(ref scanBaseRef, offset); - Vector128 prior = Unsafe.As>(ref scanRef); - Vector128 up = Unsafe.As>(ref Unsafe.Add(ref prevBaseRef, offset)); - - Unsafe.As>(ref scanRef) = AdvSimd.Add(prior, up); - - offset += bytesPerBatch; - rb -= bytesPerBatch; - } - - // Handle left over. - for (nuint i = offset; i < (uint)scanline.Length; i++) - { - ref byte scan = ref Unsafe.Add(ref scanBaseRef, offset); - byte above = Unsafe.Add(ref prevBaseRef, offset); - scan = (byte)(scan + above); - offset++; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void DecodeScalar(Span scanline, Span previousScanline) - { - ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline); - ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline); - - // Up(x) + Prior(x) - for (nuint x = 1; x < (uint)scanline.Length; x++) - { - ref byte scan = ref Unsafe.Add(ref scanBaseRef, x); - byte above = Unsafe.Add(ref prevBaseRef, x); - scan = (byte)(scan + above); - } + // The leading filter byte is metadata; every remaining byte is the modulo-256 sum of Raw(x) and Prior(x). + TensorPrimitives_.Add(scanline[1..], previousScanline[1..], scanline[1..]); } /// diff --git a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs index accfea948..7c3562cb2 100644 --- a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs +++ b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs @@ -361,34 +361,9 @@ internal class AlphaDecoder : IDisposable { HorizontalUnfilter(null, input, dst, width); } - else if (Vector256.IsHardwareAccelerated) - { - ref byte inputRef = ref MemoryMarshal.GetReference(input); - ref byte prevRef = ref MemoryMarshal.GetReference(prev); - ref byte dstRef = ref MemoryMarshal.GetReference(dst); - - nuint i; - int maxPos = width & ~31; - for (i = 0; i < (uint)maxPos; i += 32) - { - Vector256 a0 = Unsafe.As>(ref Unsafe.Add(ref inputRef, i)); - Vector256 b0 = Unsafe.As>(ref Unsafe.Add(ref prevRef, i)); - Vector256 c0 = a0.AsByte() + b0.AsByte(); - ref byte outputRef = ref Unsafe.Add(ref dstRef, i); - Unsafe.As>(ref outputRef) = c0; - } - - for (; i < (uint)width; i++) - { - Unsafe.Add(ref dstRef, i) = (byte)(Unsafe.Add(ref prevRef, i) + Unsafe.Add(ref inputRef, i)); - } - } else { - for (int i = 0; i < width; i++) - { - dst[i] = (byte)(prev[i] + input[i]); - } + TensorPrimitives_.Add(input[..width], prev[..width], dst[..width]); } } diff --git a/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs b/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs index 03bedfe67..fc3ecdd94 100644 --- a/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs +++ b/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs @@ -3,9 +3,7 @@ using System.Buffers; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Webp.Lossless; @@ -542,48 +540,7 @@ internal abstract unsafe class Vp8LHistogram DebugGuard.MustBeGreaterThanOrEqualTo(b.Length, count, nameof(b.Length)); DebugGuard.MustBeGreaterThanOrEqualTo(output.Length, count, nameof(output.Length)); - if (Avx2.IsSupported && count >= 32) - { - ref uint aRef = ref MemoryMarshal.GetReference(a); - ref uint bRef = ref MemoryMarshal.GetReference(b); - ref uint outputRef = ref MemoryMarshal.GetReference(output); - - nuint idx = 0; - do - { - // Load values. - Vector256 a0 = Unsafe.As>(ref Unsafe.Add(ref aRef, idx + 0)); - Vector256 a1 = Unsafe.As>(ref Unsafe.Add(ref aRef, idx + 8)); - Vector256 a2 = Unsafe.As>(ref Unsafe.Add(ref aRef, idx + 16)); - Vector256 a3 = Unsafe.As>(ref Unsafe.Add(ref aRef, idx + 24)); - Vector256 b0 = Unsafe.As>(ref Unsafe.Add(ref bRef, idx + 0)); - Vector256 b1 = Unsafe.As>(ref Unsafe.Add(ref bRef, idx + 8)); - Vector256 b2 = Unsafe.As>(ref Unsafe.Add(ref bRef, idx + 16)); - Vector256 b3 = Unsafe.As>(ref Unsafe.Add(ref bRef, idx + 24)); - - // Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But - // that's ok since the histogram values are less than 1<<28 (max picture count). - Unsafe.As>(ref Unsafe.Add(ref outputRef, idx + 0)) = Avx2.Add(a0, b0); - Unsafe.As>(ref Unsafe.Add(ref outputRef, idx + 8)) = Avx2.Add(a1, b1); - Unsafe.As>(ref Unsafe.Add(ref outputRef, idx + 16)) = Avx2.Add(a2, b2); - Unsafe.As>(ref Unsafe.Add(ref outputRef, idx + 24)) = Avx2.Add(a3, b3); - idx += 32; - } - while (idx <= (uint)count - 32); - - int i = (int)idx; - for (; i < count; i++) - { - output[i] = a[i] + b[i]; - } - } - else - { - for (int i = 0; i < count; i++) - { - output[i] = a[i] + b[i]; - } - } + TensorPrimitives_.Add(a[..count], b[..count], output[..count]); } } diff --git a/tests/ImageSharp.Benchmarks/General/BasicMath/AddSpan.cs b/tests/ImageSharp.Benchmarks/General/BasicMath/AddSpan.cs new file mode 100644 index 000000000..14802f590 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/BasicMath/AddSpan.cs @@ -0,0 +1,65 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath; + +public class AddSpan +{ + private byte[] scalarValues = null!; + private byte[] tensorValues = null!; + private byte[] addends = null!; + + /// + /// Gets or sets the number of values to add. + /// + [Params(32, 257, 2048)] + public int Length { get; set; } + + /// + /// Creates equivalent deterministic inputs for both implementations. + /// + [GlobalSetup] + public void Setup() + { + this.scalarValues = new byte[this.Length]; + this.tensorValues = new byte[this.Length]; + this.addends = new byte[this.Length]; + + for (int i = 0; i < this.Length; i++) + { + byte value = (byte)((i * 17) + 31); + this.scalarValues[i] = value; + this.tensorValues[i] = value; + this.addends[i] = (byte)((i * 29) + 7); + } + } + + /// + /// Adds the values with a scalar loop. + /// + /// The first result, which keeps the mutated data observable to the benchmark harness. + [Benchmark(Baseline = true)] + public byte Scalar() + { + for (int i = 0; i < this.scalarValues.Length; i++) + { + this.scalarValues[i] += this.addends[i]; + } + + return this.scalarValues[0]; + } + + /// + /// Adds the values with the tensor compatibility pipeline. + /// + /// The first result, which keeps the mutated data observable to the benchmark harness. + [Benchmark] + public byte TensorPipeline() + { + TensorPrimitives_.Add(this.tensorValues, this.addends, this.tensorValues); + return this.tensorValues[0]; + } +} diff --git a/tests/ImageSharp.Benchmarks/General/BasicMath/NormalizeSpan.cs b/tests/ImageSharp.Benchmarks/General/BasicMath/NormalizeSpan.cs new file mode 100644 index 000000000..bc11fbaff --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/BasicMath/NormalizeSpan.cs @@ -0,0 +1,61 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; + +namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath; + +public class NormalizeSpan +{ + private float[] scalarValues = null!; + private float[] tensorValues = null!; + + /// + /// Gets or sets the number of values to normalize. + /// + [Params(7, 32, 257, 2048)] + public int Length { get; set; } + + /// + /// Creates equivalent deterministic inputs for both implementations. + /// + [GlobalSetup] + public void Setup() + { + this.scalarValues = new float[this.Length]; + this.tensorValues = new float[this.Length]; + + for (int i = 0; i < this.scalarValues.Length; i++) + { + float value = ((i * 17) % 251) + 1; + this.scalarValues[i] = value; + this.tensorValues[i] = value; + } + } + + /// + /// Normalizes the values with a scalar loop. + /// + /// The first result, which keeps the mutated data observable to the benchmark harness. + [Benchmark(Baseline = true)] + public float Scalar() + { + for (int i = 0; i < this.scalarValues.Length; i++) + { + this.scalarValues[i] /= 4096F; + } + + return this.scalarValues[0]; + } + + /// + /// Normalizes the values with the tensor compatibility pipeline. + /// + /// The first result, which keeps the mutated data observable to the benchmark harness. + [Benchmark] + public float TensorPipeline() + { + Numerics.Normalize(this.tensorValues, 4096F); + return this.tensorValues[0]; + } +} diff --git a/tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssemblyComparison.cs b/tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssemblyComparison.cs new file mode 100644 index 000000000..08ac3d09b --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssemblyComparison.cs @@ -0,0 +1,813 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath; + +#pragma warning disable SA1649 // File name should match first type name +public class TensorPrimitivesJpegMultiplyAssemblyComparison +#pragma warning restore SA1649 // File name should match first type name +{ + private readonly float multiplier = -1F; + private float[] legacyValues = null!; + private float[] tensorValues = null!; + + /// + /// Creates equivalent stable inputs for both implementations. + /// + [GlobalSetup] + public void Setup() + { + this.legacyValues = new float[256]; + this.tensorValues = new float[256]; + + for (int i = 0; i < this.legacyValues.Length; i++) + { + float value = ((i * 17) % 251) + 1; + this.legacyValues[i] = value; + this.tensorValues[i] = value; + } + } + + /// + /// Multiplies the row with the retired JPEG AVX pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public float Legacy() + { + LegacyMultiply(this.legacyValues, this.multiplier); + return this.legacyValues[0]; + } + + /// + /// Multiplies the row with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float Tensor() + { + TensorPrimitives_.Multiply(this.tensorValues, this.multiplier, this.tensorValues); + return this.tensorValues[0]; + } + + /// + /// Reproduces the retired JPEG multiplication loop for assembly comparison. + /// + /// The row to multiply. + /// The scalar multiplier. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void LegacyMultiply(Span target, float multiplier) + { + ref Vector256 targetVector = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); + nuint count = (uint)target.Length / (uint)Vector256.Count; + Vector256 multiplierVector = Vector256.Create(multiplier); + + for (nuint i = 0; i < count; i++) + { + Unsafe.Add(ref targetVector, i) = Avx.Multiply(Unsafe.Add(ref targetVector, i), multiplierVector); + } + } +} + +public class TensorPrimitivesNormalizeAssemblyComparison +{ + private readonly float divisor = -1F; + private float[] legacyValues = null!; + private float[] tensorValues = null!; + + /// + /// Creates equivalent stable inputs for both implementations. + /// + [GlobalSetup] + public void Setup() + { + this.legacyValues = new float[7]; + this.tensorValues = new float[7]; + + for (int i = 0; i < this.legacyValues.Length; i++) + { + float value = ((i * 17) % 251) + 1; + this.legacyValues[i] = value; + this.tensorValues[i] = value; + } + } + + /// + /// Normalizes the values with the retired fixed-width pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public float Legacy() + { + LegacyNormalize(this.legacyValues, this.divisor); + return this.legacyValues[0]; + } + + /// + /// Normalizes the values with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float Tensor() + { + Numerics.Normalize(this.tensorValues, this.divisor); + return this.tensorValues[0]; + } + + /// + /// Reproduces the retired normalization loop for assembly comparison. + /// + /// The values to normalize. + /// The scalar divisor. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void LegacyNormalize(Span span, float sum) + { + ref float start = ref MemoryMarshal.GetReference(span); + ref float vectorEnd = ref Unsafe.Add(ref start, span.Length & ~7); + Vector256 sum256 = Vector256.Create(sum); + + while (Unsafe.IsAddressLessThan(ref start, ref vectorEnd)) + { + Unsafe.As>(ref start) /= sum256; + start = ref Unsafe.Add(ref start, (nuint)8); + } + + if ((span.Length & 7) >= 4) + { + Unsafe.As>(ref start) /= sum256.GetLower(); + start = ref Unsafe.Add(ref start, (nuint)4); + } + + ref float end = ref Unsafe.Add(ref start, span.Length & 3); + + while (Unsafe.IsAddressLessThan(ref start, ref end)) + { + start /= sum; + start = ref Unsafe.Add(ref start, (nuint)1); + } + } +} + +public class TensorPrimitivesUInt32AssemblyComparison +{ + private uint[] x = null!; + private uint[] y = null!; + private uint[] legacyDestination = null!; + private uint[] tensorDestination = null!; + + /// + /// Creates deterministic histogram inputs and independent destinations. + /// + [GlobalSetup] + public void Setup() + { + this.x = new uint[2048]; + this.y = new uint[2048]; + this.legacyDestination = new uint[2048]; + this.tensorDestination = new uint[2048]; + + for (int i = 0; i < this.x.Length; i++) + { + this.x[i] = (uint)((i * 17) + 31); + this.y[i] = (uint)((i * 29) + 7); + } + } + + /// + /// Adds histogram bins with the retired four-vector AVX2 pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public uint Legacy() + { + LegacyAdd(this.x, this.y, this.legacyDestination); + return this.legacyDestination[0]; + } + + /// + /// Adds histogram bins with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public uint Tensor() + { + TensorPrimitives_.Add(this.x, this.y, this.tensorDestination); + return this.tensorDestination[0]; + } + + /// + /// Reproduces the retired WebP histogram addition loop for assembly comparison. + /// + /// The first histogram. + /// The second histogram. + /// The destination receiving the sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void LegacyAdd(ReadOnlySpan x, ReadOnlySpan y, Span destination) + { + ref uint xRef = ref MemoryMarshal.GetReference(x); + ref uint yRef = ref MemoryMarshal.GetReference(y); + ref uint destinationRef = ref MemoryMarshal.GetReference(destination); + + nuint index = 0; + + do + { + Vector256 x0 = Unsafe.As>(ref Unsafe.Add(ref xRef, index)); + Vector256 x1 = Unsafe.As>(ref Unsafe.Add(ref xRef, index + 8)); + Vector256 x2 = Unsafe.As>(ref Unsafe.Add(ref xRef, index + 16)); + Vector256 x3 = Unsafe.As>(ref Unsafe.Add(ref xRef, index + 24)); + Vector256 y0 = Unsafe.As>(ref Unsafe.Add(ref yRef, index)); + Vector256 y1 = Unsafe.As>(ref Unsafe.Add(ref yRef, index + 8)); + Vector256 y2 = Unsafe.As>(ref Unsafe.Add(ref yRef, index + 16)); + Vector256 y3 = Unsafe.As>(ref Unsafe.Add(ref yRef, index + 24)); + + Unsafe.As>(ref Unsafe.Add(ref destinationRef, index)) = Avx2.Add(x0, y0); + Unsafe.As>(ref Unsafe.Add(ref destinationRef, index + 8)) = Avx2.Add(x1, y1); + Unsafe.As>(ref Unsafe.Add(ref destinationRef, index + 16)) = Avx2.Add(x2, y2); + Unsafe.As>(ref Unsafe.Add(ref destinationRef, index + 24)) = Avx2.Add(x3, y3); + index += 32; + } + while (index <= (uint)x.Length - 32); + + for (int i = (int)index; i < x.Length; i++) + { + destination[i] = x[i] + y[i]; + } + } +} + +public class TensorPrimitivesByteAssemblyComparison +{ + private byte[] x = null!; + private byte[] y = null!; + private byte[] legacyDestination = null!; + private byte[] tensorDestination = null!; + + /// + /// Creates deterministic byte inputs and independent destinations. + /// + [GlobalSetup] + public void Setup() + { + this.x = new byte[2048]; + this.y = new byte[2048]; + this.legacyDestination = new byte[2048]; + this.tensorDestination = new byte[2048]; + + for (int i = 0; i < this.x.Length; i++) + { + this.x[i] = (byte)((i * 17) + 31); + this.y[i] = (byte)((i * 29) + 7); + } + } + + /// + /// Adds bytes with the retired WebP AVX2 pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public byte Legacy() + { + LegacyAdd(this.x, this.y, this.legacyDestination); + return this.legacyDestination[0]; + } + + /// + /// Adds bytes with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public byte Tensor() + { + TensorPrimitives_.Add(this.x, this.y, this.tensorDestination); + return this.tensorDestination[0]; + } + + /// + /// Reproduces the retired WebP byte addition loop for assembly comparison. + /// + /// The first input. + /// The second input. + /// The destination receiving modulo-256 sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void LegacyAdd(ReadOnlySpan x, ReadOnlySpan y, Span destination) + { + ref byte xRef = ref MemoryMarshal.GetReference(x); + ref byte yRef = ref MemoryMarshal.GetReference(y); + ref byte destinationRef = ref MemoryMarshal.GetReference(destination); + + nuint i; + int maxPosition = x.Length & ~31; + + for (i = 0; i < (uint)maxPosition; i += 32) + { + Vector256 x0 = Unsafe.As>(ref Unsafe.Add(ref xRef, i)); + Vector256 y0 = Unsafe.As>(ref Unsafe.Add(ref yRef, i)); + Vector256 result = x0.AsByte() + y0.AsByte(); + Unsafe.As>(ref Unsafe.Add(ref destinationRef, i)) = result; + } + + for (; i < (uint)x.Length; i++) + { + Unsafe.Add(ref destinationRef, i) = (byte)(Unsafe.Add(ref xRef, i) + Unsafe.Add(ref yRef, i)); + } + } +} + +public class TensorPrimitivesSingleAddAssemblyComparison +{ + private float[] legacyTarget = null!; + private float[] tensorTarget = null!; + private float[] source = null!; + + /// + /// Creates deterministic JPEG row inputs. + /// + [GlobalSetup] + public void Setup() + { + this.legacyTarget = new float[2048]; + this.tensorTarget = new float[2048]; + this.source = new float[2048]; + + for (int i = 0; i < this.source.Length; i++) + { + float value = ((i * 17) % 251) + 1; + this.legacyTarget[i] = value; + this.tensorTarget[i] = value; + this.source[i] = ((i * 29) % 31) - 15; + } + } + + /// + /// Adds JPEG row values with the retired AVX pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public float Legacy() + { + LegacyAdd(this.legacyTarget, this.source); + return this.legacyTarget[0]; + } + + /// + /// Adds JPEG row values with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float Tensor() + { + TensorPrimitives_.Add(this.tensorTarget, this.source, this.tensorTarget); + return this.tensorTarget[0]; + } + + /// + /// Reproduces the retired JPEG row addition loop for assembly comparison. + /// + /// The destination row. + /// The row added to the destination. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void LegacyAdd(Span target, ReadOnlySpan source) + { + ref Vector256 targetVector = ref Unsafe.As>(ref MemoryMarshal.GetReference(target)); + ref Vector256 sourceVector = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); + nuint count = (uint)source.Length / (uint)Vector256.Count; + + for (nuint i = 0; i < count; i++) + { + Unsafe.Add(ref targetVector, i) = Avx.Add(Unsafe.Add(ref targetVector, i), Unsafe.Add(ref sourceVector, i)); + } + } +} + +[GenericTypeArguments(typeof(byte))] +[GenericTypeArguments(typeof(uint))] +[GenericTypeArguments(typeof(int))] +[GenericTypeArguments(typeof(float))] +[GenericTypeArguments(typeof(double))] +public class TensorPrimitivesClampAssemblyComparison + where T : unmanaged, INumber +{ + private T[] legacyValues = null!; + private T[] tensorValues = null!; + private T min; + private T max; + + /// + /// Creates deterministic clamp inputs for the current element type. + /// + [GlobalSetup] + public void Setup() + { + this.legacyValues = new T[2048]; + this.tensorValues = new T[2048]; + this.min = T.CreateTruncating(64); + this.max = T.CreateTruncating(128); + + for (int i = 0; i < this.legacyValues.Length; i++) + { + T value = T.CreateTruncating((i * 31) % 257); + this.legacyValues[i] = value; + this.tensorValues[i] = value; + } + } + + /// + /// Clamps values with the retired pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public T Legacy() + { + LegacyClamp(this.legacyValues, this.min, this.max); + return this.legacyValues[0]; + } + + /// + /// Clamps values with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public T Tensor() + { + TensorPrimitives_.Clamp(this.tensorValues, this.min, this.max, this.tensorValues); + return this.tensorValues[0]; + } + + /// + /// Reproduces the retired clamp pipeline for assembly comparison. + /// + /// The values to clamp. + /// The inclusive lower bound. + /// The inclusive upper bound. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void LegacyClamp(Span span, T min, T max) + { + int remainder = Numerics.ModuloP2(span.Length, Vector.Count); + int adjustedCount = span.Length - remainder; + + if (adjustedCount > 0) + { + Vector vectorMin = new(min); + Vector vectorMax = new(max); + nint vectorCount = (nint)(uint)adjustedCount / Vector.Count; + nint remainingVectors = Numerics.Modulo4(vectorCount); + nint unrolledVectors = vectorCount - remainingVectors; + + ref Vector current0 = ref Unsafe.As>(ref MemoryMarshal.GetReference(span)); + ref Vector current1 = ref Unsafe.Add(ref current0, 1); + ref Vector current2 = ref Unsafe.Add(ref current0, 2); + ref Vector current3 = ref Unsafe.Add(ref current0, 3); + ref Vector end = ref Unsafe.Add(ref current0, unrolledVectors); + + while (Unsafe.IsAddressLessThan(ref current0, ref end)) + { + current0 = Vector.Min(Vector.Max(vectorMin, current0), vectorMax); + current1 = Vector.Min(Vector.Max(vectorMin, current1), vectorMax); + current2 = Vector.Min(Vector.Max(vectorMin, current2), vectorMax); + current3 = Vector.Min(Vector.Max(vectorMin, current3), vectorMax); + + current0 = ref Unsafe.Add(ref current0, 4); + current1 = ref Unsafe.Add(ref current1, 4); + current2 = ref Unsafe.Add(ref current2, 4); + current3 = ref Unsafe.Add(ref current3, 4); + } + + if (remainingVectors > 0) + { + current0 = ref end; + end = ref Unsafe.Add(ref end, remainingVectors); + + while (Unsafe.IsAddressLessThan(ref current0, ref end)) + { + current0 = Vector.Min(Vector.Max(vectorMin, current0), vectorMax); + current0 = ref Unsafe.Add(ref current0, 1); + } + } + } + + for (int i = adjustedCount; i < span.Length; i++) + { + T value = span[i]; + span[i] = value > max ? max : value < min ? min : value; + } + } +} + +public class TensorPrimitivesIccMaxAssemblyComparison +{ + private Vector4[] legacyValues = null!; + private Vector4[] tensorValues = null!; + + /// + /// Creates deterministic ICC values containing positive and negative channels. + /// + [GlobalSetup] + public void Setup() + { + this.legacyValues = new Vector4[512]; + this.tensorValues = new Vector4[512]; + + for (int i = 0; i < this.legacyValues.Length; i++) + { + float value = ((i * 17) % 251) - 125; + Vector4 vector = new(value, value + 1, value - 1, value + 2); + this.legacyValues[i] = vector; + this.tensorValues[i] = vector; + } + } + + /// + /// Clips negative channels with the retired ICC pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public float Legacy() + { + for (int i = 0; i < this.legacyValues.Length; i++) + { + this.legacyValues[i] = Vector4.Max(this.legacyValues[i], Vector4.Zero); + } + + return this.legacyValues[0].X; + } + + /// + /// Clips negative channels with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float Tensor() + { + Span values = MemoryMarshal.Cast(this.tensorValues.AsSpan()); + TensorPrimitives_.Max(values, 0F, values); + return values[0]; + } +} + +public class TensorPrimitivesIccMultiplyAssemblyComparison +{ + private readonly float multiplier = 65280F / 65535F; + private Vector4[] source = null!; + private Vector4[] legacyDestination = null!; + private Vector4[] tensorDestination = null!; + + /// + /// Creates deterministic ICC inputs and independent destinations. + /// + [GlobalSetup] + public void Setup() + { + this.source = new Vector4[512]; + this.legacyDestination = new Vector4[512]; + this.tensorDestination = new Vector4[512]; + + for (int i = 0; i < this.source.Length; i++) + { + float value = ((i * 17) % 251) + 1; + this.source[i] = new Vector4(value, value + 1, value + 2, value + 3); + } + } + + /// + /// Multiplies ICC channels with the retired pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public float Legacy() + { + Span source = MemoryMarshal.Cast(this.source.AsSpan()); + Span destination = MemoryMarshal.Cast(this.legacyDestination.AsSpan()); + ref Vector sourceVector = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); + ref Vector destinationVector = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); + Vector scale = new(this.multiplier); + nuint count = (uint)source.Length / (uint)Vector.Count; + + for (nuint i = 0; i < count; i++) + { + Unsafe.Add(ref destinationVector, i) = Unsafe.Add(ref sourceVector, i) * scale; + } + + return destination[0]; + } + + /// + /// Multiplies ICC channels with the tensor compatibility pipeline. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float Tensor() + { + Span source = MemoryMarshal.Cast(this.source.AsSpan()); + Span destination = MemoryMarshal.Cast(this.tensorDestination.AsSpan()); + TensorPrimitives_.Multiply(source, this.multiplier, destination); + return destination[0]; + } +} + +#if NET10_0_OR_GREATER +[GenericTypeArguments(typeof(byte))] +[GenericTypeArguments(typeof(uint))] +[GenericTypeArguments(typeof(float))] +public class TensorPrimitivesRuntimeAddAssemblyComparison + where T : unmanaged, INumber +{ + private T[] x = null!; + private T[] y = null!; + private T[] compatibilityDestination = null!; + private T[] runtimeDestination = null!; + + /// + /// Creates deterministic inputs and independent destinations. + /// + [GlobalSetup] + public void Setup() + { + this.x = new T[2048]; + this.y = new T[2048]; + this.compatibilityDestination = new T[2048]; + this.runtimeDestination = new T[2048]; + + for (int i = 0; i < this.x.Length; i++) + { + this.x[i] = T.CreateTruncating((i * 17) + 31); + this.y[i] = T.CreateTruncating((i * 29) + 7); + } + } + + /// + /// Adds values with the compatibility implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public T Compatibility() + { + TensorPrimitives_.Add(this.x, this.y, this.compatibilityDestination); + return this.compatibilityDestination[0]; + } + + /// + /// Adds values with the .NET runtime implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public T Runtime() + { + System.Numerics.Tensors.TensorPrimitives.Add(this.x, this.y, this.runtimeDestination); + return this.runtimeDestination[0]; + } +} + +[GenericTypeArguments(typeof(byte))] +[GenericTypeArguments(typeof(uint))] +[GenericTypeArguments(typeof(int))] +[GenericTypeArguments(typeof(float))] +[GenericTypeArguments(typeof(double))] +public class TensorPrimitivesRuntimeClampAssemblyComparison + where T : unmanaged, INumber +{ + private T[] compatibilityValues = null!; + private T[] runtimeValues = null!; + private T min; + private T max; + + /// + /// Creates deterministic inputs for both implementations. + /// + [GlobalSetup] + public void Setup() + { + this.compatibilityValues = new T[2048]; + this.runtimeValues = new T[2048]; + this.min = T.CreateTruncating(64); + this.max = T.CreateTruncating(128); + + for (int i = 0; i < this.compatibilityValues.Length; i++) + { + T value = T.CreateTruncating((i * 31) % 257); + this.compatibilityValues[i] = value; + this.runtimeValues[i] = value; + } + } + + /// + /// Clamps values with the compatibility implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark(Baseline = true)] + public T Compatibility() + { + TensorPrimitives_.Clamp(this.compatibilityValues, this.min, this.max, this.compatibilityValues); + return this.compatibilityValues[0]; + } + + /// + /// Clamps values with the .NET runtime implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public T Runtime() + { + System.Numerics.Tensors.TensorPrimitives.Clamp(this.runtimeValues, this.min, this.max, this.runtimeValues); + return this.runtimeValues[0]; + } +} + +public class TensorPrimitivesRuntimeSingleScalarAssemblyComparison +{ + private readonly float scalar = -1F; + private float[] compatibilityValues = null!; + private float[] runtimeValues = null!; + + /// + /// Creates equivalent stable inputs for both implementations. + /// + [GlobalSetup] + public void Setup() + { + this.compatibilityValues = new float[2048]; + this.runtimeValues = new float[2048]; + + for (int i = 0; i < this.compatibilityValues.Length; i++) + { + float value = ((i * 17) % 251) + 1; + this.compatibilityValues[i] = value; + this.runtimeValues[i] = value; + } + } + + /// + /// Divides values with the compatibility implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float CompatibilityDivide() + { + TensorPrimitives_.Divide(this.compatibilityValues, this.scalar, this.compatibilityValues); + return this.compatibilityValues[0]; + } + + /// + /// Divides values with the .NET runtime implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float RuntimeDivide() + { + System.Numerics.Tensors.TensorPrimitives.Divide(this.runtimeValues, this.scalar, this.runtimeValues); + return this.runtimeValues[0]; + } + + /// + /// Computes maximum values with the compatibility implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float CompatibilityMax() + { + TensorPrimitives_.Max(this.compatibilityValues, 0F, this.compatibilityValues); + return this.compatibilityValues[0]; + } + + /// + /// Computes maximum values with the .NET runtime implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float RuntimeMax() + { + System.Numerics.Tensors.TensorPrimitives.Max(this.runtimeValues, 0F, this.runtimeValues); + return this.runtimeValues[0]; + } + + /// + /// Multiplies values with the compatibility implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float CompatibilityMultiply() + { + TensorPrimitives_.Multiply(this.compatibilityValues, this.scalar, this.compatibilityValues); + return this.compatibilityValues[0]; + } + + /// + /// Multiplies values with the .NET runtime implementation. + /// + /// The first result, which keeps the writes observable to the benchmark harness. + [Benchmark] + public float RuntimeMultiply() + { + System.Numerics.Tensors.TensorPrimitives.Multiply(this.runtimeValues, this.scalar, this.runtimeValues); + return this.runtimeValues[0]; + } +} +#endif diff --git a/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj b/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj index fa5fdd816..0351ecae9 100644 --- a/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj +++ b/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj @@ -69,6 +69,7 @@ + diff --git a/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs b/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs new file mode 100644 index 000000000..1cc5b943f --- /dev/null +++ b/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs @@ -0,0 +1,356 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Tests.Common; + +public class TensorPrimitivesTests +{ + /// + /// Gets lengths that exercise scalar execution, every SIMD width, overlapping tails, and the unrolled loop. + /// + public static TheoryData SpanLengths => new() + { + 0, + 1, + 3, + 4, + 5, + 7, + 8, + 9, + 15, + 16, + 17, + 31, + 32, + 33, + 63, + 64, + 65, + 127, + 128, + 129, + 2048 + }; + + /// + /// Verifies that byte addition wraps modulo 256 and supports either input as the in-place destination. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void AddByteMatchesScalarFormula(int length) + { + byte[] x = new byte[length]; + byte[] y = new byte[length]; + byte[] expected = new byte[length]; + + for (int i = 0; i < length; i++) + { + x[i] = (byte)((i * 23) + 197); + y[i] = (byte)((i * 41) + 113); + expected[i] = unchecked((byte)(x[i] + y[i])); + } + + byte[] destination = new byte[length]; + TensorPrimitives_.Add(x, y, destination); + Assert.Equal(expected, destination); + + byte[] xInPlace = (byte[])x.Clone(); + TensorPrimitives_.Add(xInPlace, y, xInPlace); + Assert.Equal(expected, xInPlace); + + byte[] yInPlace = (byte[])y.Clone(); + TensorPrimitives_.Add(x, yInPlace, yInPlace); + Assert.Equal(expected, yInPlace); + } + + /// + /// Verifies that unsigned integer addition preserves unchecked histogram accumulation semantics. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void AddUInt32MatchesScalarFormula(int length) + { + uint[] x = new uint[length]; + uint[] y = new uint[length]; + uint[] expected = new uint[length]; + + for (int i = 0; i < length; i++) + { + x[i] = ((uint)i * 1_234_567U) + 0xF0000000U; + y[i] = ((uint)i * 7_654_321U) + 0x30000000U; + expected[i] = unchecked(x[i] + y[i]); + } + + TensorPrimitives_.Add(x, y, x); + Assert.Equal(expected, x); + } + + /// + /// Verifies that integer clamping produces identical results for separate and in-place destinations. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void ClampInt32MatchesScalarFormula(int length) + { + int[] source = new int[length]; + int[] expected = new int[length]; + + for (int i = 0; i < source.Length; i++) + { + source[i] = ((i * 37) % 401) - 200; + expected[i] = Math.Clamp(source[i], -73, 91); + } + + int[] destination = new int[length]; + TensorPrimitives_.Clamp(source, -73, 91, destination); + Assert.Equal(expected, destination); + + int[] inPlace = (int[])source.Clone(); + TensorPrimitives_.Clamp(inPlace, -73, 91, inPlace); + Assert.Equal(expected, inPlace); + } + + /// + /// Verifies that floating-point clamping matches the runtime tensor formula for special values and unordered bounds. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void ClampSingleMatchesRuntimeFormula(int length) + { + float[] values = + { + float.NaN, + -0F, + 0F, + -1F, + 1F, + float.NegativeInfinity, + float.PositiveInfinity + }; + + float[] source = new float[length]; + float[] expected = new float[length]; + + for (int i = 0; i < source.Length; i++) + { + source[i] = values[i % values.Length]; + + // Runtime main follows Min(Max(x, min), max) for vectorizable types, including unordered bounds. + expected[i] = float.Min(float.Max(source[i], 2F), -2F); + } + + TensorPrimitives_.Clamp(source, 2F, -2F, source); + AssertSingleBitsEqual(expected, source); + } + + /// + /// Verifies that single-precision clamping preserves the runtime's signed-zero and NaN behavior. + /// + [Fact] + public void ClampSinglePreservesRuntimeSpecialValueSemantics() + { + float[] values = + { + float.NaN, + float.NegativeInfinity, + -0F, + 0F, + float.PositiveInfinity + }; + + float[] actual = new float[129]; + float[] expected = new float[actual.Length]; + + for (int i = 0; i < actual.Length; i++) + { + actual[i] = values[i % values.Length]; + expected[i] = float.Min(float.Max(actual[i], -0F), 0F); + } + + TensorPrimitives_.Clamp(actual, -0F, 0F, actual); + AssertSingleBitsEqual(expected, actual); + } + + /// + /// Verifies that double-precision clamping preserves the runtime's signed-zero and NaN behavior. + /// + [Fact] + public void ClampDoublePreservesRuntimeSpecialValueSemantics() + { + double[] values = + { + double.NaN, + double.NegativeInfinity, + -0D, + 0D, + double.PositiveInfinity + }; + + double[] actual = new double[65]; + double[] expected = new double[actual.Length]; + + for (int i = 0; i < actual.Length; i++) + { + actual[i] = values[i % values.Length]; + expected[i] = double.Min(double.Max(actual[i], -0D), 0D); + } + + TensorPrimitives_.Clamp(actual, -0D, 0D, actual); + AssertDoubleBitsEqual(expected, actual); + } + + /// + /// Verifies that division produces identical results for separate and in-place destinations. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void DivideSingleMatchesScalarFormula(int length) + { + float[] source = new float[length]; + float[] expected = new float[length]; + + for (int i = 0; i < source.Length; i++) + { + source[i] = (i - 65.25F) * 1.75F; + expected[i] = source[i] / 3.25F; + } + + float[] destination = new float[length]; + TensorPrimitives_.Divide(source, 3.25F, destination); + AssertSingleBitsEqual(expected, destination); + + float[] inPlace = (float[])source.Clone(); + TensorPrimitives_.Divide(inPlace, 3.25F, inPlace); + AssertSingleBitsEqual(expected, inPlace); + } + + /// + /// Verifies that maximum selection preserves the runtime's NaN and signed-zero semantics. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void MaxSingleMatchesRuntimeFormula(int length) + { + float[] values = + { + float.NaN, + float.NegativeInfinity, + -1F, + -0F, + 0F, + 1F, + float.PositiveInfinity + }; + + float[] actual = new float[length]; + float[] expected = new float[length]; + + for (int i = 0; i < length; i++) + { + actual[i] = values[i % values.Length]; + expected[i] = float.Max(actual[i], -0F); + } + + TensorPrimitives_.Max(actual, -0F, actual); + AssertSingleBitsEqual(expected, actual); + } + + /// + /// Verifies that multiplication produces identical results for separate and in-place destinations. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void MultiplySingleMatchesScalarFormula(int length) + { + float[] source = new float[length]; + float[] expected = new float[length]; + + for (int i = 0; i < source.Length; i++) + { + source[i] = (i - 65.25F) * 1.75F; + expected[i] = source[i] * 0.375F; + } + + float[] destination = new float[length]; + TensorPrimitives_.Multiply(source, 0.375F, destination); + AssertSingleBitsEqual(expected, destination); + + TensorPrimitives_.Multiply(source, 0.375F, source); + AssertSingleBitsEqual(expected, source); + } + + /// + /// Verifies that the normalization compatibility call preserves its element-wise division contract. + /// + /// The input length. + [Theory] + [MemberData(nameof(SpanLengths))] + public void NormalizeMatchesScalarFormula(int length) + { + float[] actual = new float[length]; + float[] expected = new float[length]; + + for (int i = 0; i < actual.Length; i++) + { + actual[i] = (i + 1) * 0.125F; + expected[i] = actual[i] / 7.5F; + } + + Numerics.Normalize(actual, 7.5F); + AssertSingleBitsEqual(expected, actual); + } + + /// + /// Compares floating-point results while preserving signed-zero behavior. + /// + /// The expected values. + /// The actual values. + private static void AssertSingleBitsEqual(ReadOnlySpan expected, ReadOnlySpan actual) + { + Assert.Equal(expected.Length, actual.Length); + + for (int i = 0; i < expected.Length; i++) + { + if (float.IsNaN(expected[i])) + { + Assert.True(float.IsNaN(actual[i])); + } + else + { + Assert.Equal(BitConverter.SingleToInt32Bits(expected[i]), BitConverter.SingleToInt32Bits(actual[i])); + } + } + } + + /// + /// Compares double-precision results while preserving signed-zero behavior. + /// + /// The expected values. + /// The actual values. + private static void AssertDoubleBitsEqual(ReadOnlySpan expected, ReadOnlySpan actual) + { + Assert.Equal(expected.Length, actual.Length); + + for (int i = 0; i < expected.Length; i++) + { + if (double.IsNaN(expected[i])) + { + Assert.True(double.IsNaN(actual[i])); + } + else + { + Assert.Equal(BitConverter.DoubleToInt64Bits(expected[i]), BitConverter.DoubleToInt64Bits(actual[i])); + } + } + } +}