From 979a9237c59e0087c2ecd23b4f2eac9f2df4b823 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sun, 26 Jul 2026 03:07:53 +1000 Subject: [PATCH] Align tensor primitive dispatch with runtime --- .../Common/Helpers/TensorPrimitives_.Add.cs | 84 ++ .../Common/Helpers/TensorPrimitives_.Clamp.cs | 318 ++++++++ .../Helpers/TensorPrimitives_.Divide.cs | 83 ++ ...itives.cs => TensorPrimitives_.Helpers.cs} | 768 +----------------- .../Common/Helpers/TensorPrimitives_.Max.cs | 245 ++++++ .../Helpers/TensorPrimitives_.Multiply.cs | 72 ++ .../Helpers/TensorPrimitives_.Negate.cs | 5 +- 7 files changed, 811 insertions(+), 764 deletions(-) create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs rename src/ImageSharp/Common/Helpers/{TensorPrimitives.cs => TensorPrimitives_.Helpers.cs} (55%) create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs create mode 100644 src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs new file mode 100644 index 000000000..f2ef214cf --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs @@ -0,0 +1,84 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +internal static partial class TensorPrimitives_ +{ + /// + /// Computes the element-wise sum of the values in and . + /// + /// The element type. + /// The first addends. + /// The second addends. + /// The destination for the sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IAdditionOperators, IAdditiveIdentity + => InvokeSpanSpanIntoSpan>(x, y, destination); + + /// + /// Computes the element-wise sum of the values in and the scalar . + /// + /// The element type. + /// The first addends. + /// The scalar second addend. + /// The destination for the sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ReadOnlySpan x, T y, Span destination) + where T : IAdditionOperators, IAdditiveIdentity + => InvokeSpanScalarIntoSpan>(x, y, destination); + + /// + /// Adds corresponding values. + /// + /// The element type. + private readonly struct AddOperator : IBinaryOperator + where T : IAdditionOperators, IAdditiveIdentity + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Adds scalar values. + /// + /// The first addend. + /// The second addend. + /// The sum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => x + y; + + /// + /// Adds 128-bit vectors. + /// + /// The first addends. + /// The second addends. + /// The sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y; + + /// + /// Adds 256-bit vectors. + /// + /// The first addends. + /// The second addends. + /// The sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y; + + /// + /// Adds 512-bit vectors. + /// + /// The first addends. + /// The second addends. + /// The sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y; + } +} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs new file mode 100644 index 000000000..e2cc9097e --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs @@ -0,0 +1,318 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +internal static partial class TensorPrimitives_ +{ + /// + /// Computes the element-wise result of clamping to the inclusive range specified + /// by and . + /// + /// The element type. + /// The values to clamp. + /// The inclusive lower bound. + /// The inclusive upper bound. + /// The destination for the clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Clamp(ReadOnlySpan x, T min, T max, Span destination) + where T : INumber + => InvokeSpanScalarScalarIntoSpan>(x, min, max, destination); + + /// + /// Clamps single-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 ClampSingle( + Vector128 value, + Vector128 min, + Vector128 max) + { + // Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the + // first operand and select negative zero when equal values have different signs. + Vector128 maximum = Vector128.ConditionalSelect( + Vector128.LessThan(min, value) + | ~Vector128.Equals(value, value) + | (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), + value, + min); + + return Vector128.ConditionalSelect( + Vector128.LessThan(maximum, max) + | ~Vector128.Equals(maximum, maximum) + | (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), + maximum, + max); + } + + /// + /// Clamps single-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 ClampSingle( + Vector256 value, + Vector256 min, + Vector256 max) + { + Vector256 maximum = Vector256.ConditionalSelect( + Vector256.LessThan(min, value) + | ~Vector256.Equals(value, value) + | (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), + value, + min); + + return Vector256.ConditionalSelect( + Vector256.LessThan(maximum, max) + | ~Vector256.Equals(maximum, maximum) + | (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), + maximum, + max); + } + + /// + /// Clamps single-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 ClampSingle( + Vector512 value, + Vector512 min, + Vector512 max) + { + Vector512 maximum = Vector512.ConditionalSelect( + Vector512.LessThan(min, value) + | ~Vector512.Equals(value, value) + | (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), + value, + min); + + return Vector512.ConditionalSelect( + Vector512.LessThan(maximum, max) + | ~Vector512.Equals(maximum, maximum) + | (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), + maximum, + max); + } + + /// + /// Clamps double-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 ClampDouble( + Vector128 value, + Vector128 min, + Vector128 max) + { + Vector128 maximum = Vector128.ConditionalSelect( + Vector128.LessThan(min, value) + | ~Vector128.Equals(value, value) + | (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), + value, + min); + + return Vector128.ConditionalSelect( + Vector128.LessThan(maximum, max) + | ~Vector128.Equals(maximum, maximum) + | (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), + maximum, + max); + } + + /// + /// Clamps double-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 ClampDouble( + Vector256 value, + Vector256 min, + Vector256 max) + { + Vector256 maximum = Vector256.ConditionalSelect( + Vector256.LessThan(min, value) + | ~Vector256.Equals(value, value) + | (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), + value, + min); + + return Vector256.ConditionalSelect( + Vector256.LessThan(maximum, max) + | ~Vector256.Equals(maximum, maximum) + | (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), + maximum, + max); + } + + /// + /// Clamps double-precision values with the normalized runtime semantics. + /// + /// The values to clamp. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 ClampDouble( + Vector512 value, + Vector512 min, + Vector512 max) + { + Vector512 maximum = Vector512.ConditionalSelect( + Vector512.LessThan(min, value) + | ~Vector512.Equals(value, value) + | (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), + value, + min); + + return Vector512.ConditionalSelect( + Vector512.LessThan(maximum, max) + | ~Vector512.Equals(maximum, maximum) + | (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), + maximum, + max); + } + + /// + /// Clamps values using the complete runtime tensor contract, including signed-zero correction. + /// + /// The element type. + private readonly struct ClampOperator : ITernaryOperator + where T : INumber + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Clamps a scalar value. + /// + /// The value. + /// The inclusive lower bound. + /// The inclusive upper bound. + /// The clamped value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T min, T max) + => Vector128.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max); + + /// + /// Clamps a 128-bit vector. + /// + /// The values. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 min, Vector128 max) + { + if (typeof(T) == typeof(float)) + { + Vector128 result = ClampSingle( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref min), + Unsafe.As, Vector128>(ref max)); + + return Unsafe.As, Vector128>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector128 result = ClampDouble( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref min), + Unsafe.As, Vector128>(ref max)); + + return Unsafe.As, Vector128>(ref result); + } + + return Vector128_.Clamp(x, min, max); + } + + /// + /// Clamps a 256-bit vector. + /// + /// The values. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 min, Vector256 max) + { + if (typeof(T) == typeof(float)) + { + Vector256 result = ClampSingle( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref min), + Unsafe.As, Vector256>(ref max)); + + return Unsafe.As, Vector256>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector256 result = ClampDouble( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref min), + Unsafe.As, Vector256>(ref max)); + + return Unsafe.As, Vector256>(ref result); + } + + return Vector256_.Clamp(x, min, max); + } + + /// + /// Clamps a 512-bit vector. + /// + /// The values. + /// The inclusive lower bounds. + /// The inclusive upper bounds. + /// The clamped values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 min, Vector512 max) + { + if (typeof(T) == typeof(float)) + { + Vector512 result = ClampSingle( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref min), + Unsafe.As, Vector512>(ref max)); + + return Unsafe.As, Vector512>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector512 result = ClampDouble( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref min), + Unsafe.As, Vector512>(ref max)); + + return Unsafe.As, Vector512>(ref result); + } + + return Vector512_.Clamp(x, min, max); + } + } +} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs new file mode 100644 index 000000000..e40a26cea --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs @@ -0,0 +1,83 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +internal static partial class TensorPrimitives_ +{ + /// + /// Computes the element-wise result of dividing the values in by . + /// + /// The element type. + /// The dividend values. + /// The divisor. + /// The destination for the quotient values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Divide(ReadOnlySpan x, T y, Span destination) + where T : IDivisionOperators + => InvokeSpanScalarIntoSpanForDivision>(x, y, destination); + + /// + /// Determines whether has the same vector division support as . + /// + /// The element type. + /// when is a 32-bit signed native integer type. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsInt32Like() + => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); + + /// + /// Divides values by a scalar. + /// + /// The element type. + private readonly struct DivideOperator : IBinaryOperator + where T : IDivisionOperators + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => typeof(T) == typeof(float) + || typeof(T) == typeof(double) + || (Vector256.IsHardwareAccelerated && IsInt32Like()); + + /// + /// Divides scalar values. + /// + /// The dividend. + /// The divisor. + /// The quotient. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => x / y; + + /// + /// Divides 128-bit vectors. + /// + /// The dividends. + /// The divisors. + /// The quotients. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; + + /// + /// Divides 256-bit vectors. + /// + /// The dividends. + /// The divisors. + /// The quotients. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; + + /// + /// Divides 512-bit vectors. + /// + /// The dividends. + /// The divisors. + /// The quotients. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; + } +} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs similarity index 55% rename from src/ImageSharp/Common/Helpers/TensorPrimitives.cs rename to src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs index 4be4ab3aa..3ba1b61af 100644 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs @@ -15,9 +15,7 @@ namespace SixLabors.ImageSharp.Common.Helpers; /// The API shape follows System.Numerics.Tensors.TensorPrimitives so call sites can move to the runtime /// implementation when ImageSharp no longer supports target frameworks that predate it. /// -#pragma warning disable SA1649 // File name should match first type name internal static partial class TensorPrimitives_ -#pragma warning restore SA1649 // File name should match first type name { /// /// Defines an element-wise binary operation. @@ -111,80 +109,6 @@ internal static partial class TensorPrimitives_ public static abstract Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z); } - /// - /// Computes the element-wise result of clamping to the inclusive range specified - /// by and . - /// - /// The element type. - /// The values to clamp. - /// The inclusive lower bound. - /// The inclusive upper bound. - /// The destination for the clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Clamp(ReadOnlySpan x, T min, T max, Span destination) - where T : INumber - => InvokeSpanScalarScalarIntoSpan>(x, min, max, destination); - - /// - /// Computes the element-wise sum of the values in and . - /// - /// The element type. - /// The first addends. - /// The second addends. - /// The destination for the sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IAdditionOperators, IAdditiveIdentity - => InvokeSpanSpanIntoSpan>(x, y, destination); - - /// - /// Computes the element-wise sum of the values in and the scalar . - /// - /// The element type. - /// The first addends. - /// The scalar second addend. - /// The destination for the sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ReadOnlySpan x, T y, Span destination) - where T : IAdditionOperators, IAdditiveIdentity - => InvokeSpanScalarIntoSpan>(x, y, destination); - - /// - /// Computes the element-wise result of dividing the values in by . - /// - /// The element type. - /// The dividend values. - /// The divisor. - /// The destination for the quotient values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Divide(ReadOnlySpan x, T y, Span destination) - where T : IDivisionOperators - => InvokeSpanScalarIntoSpanForDivision>(x, y, destination); - - /// - /// Computes the element-wise maximum of the values in and . - /// - /// The element type. - /// The values to compare. - /// The value to compare with each element. - /// The destination for the maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Max(ReadOnlySpan x, T y, Span destination) - where T : INumber - => InvokeSpanScalarIntoSpan>(x, y, destination); - - /// - /// Computes the element-wise product of the values in and . - /// - /// The element type. - /// The multiplicands. - /// The multiplier. - /// The destination for the products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Multiply(ReadOnlySpan x, T y, Span destination) - where T : IMultiplyOperators, IMultiplicativeIdentity - => InvokeSpanScalarIntoSpan>(x, y, destination); - /// /// Performs an element-wise binary operation between two spans. /// @@ -204,15 +128,13 @@ internal static partial class TensorPrimitives_ ref T yRef = ref MemoryMarshal.GetReference(y); ref T destinationRef = ref MemoryMarshal.GetReference(destination); nuint length = (uint)x.Length; - nuint vector512Threshold = Unsafe.SizeOf() == 1 ? (uint)Vector512.Count : 512; - // Match the runtime's AVX-512 selection for byte-sized elements once one complete vector is available. - // Wider elements retain the measured crossover point where their 512-bit setup cost becomes worthwhile. + // Runtime main selects the widest supported pipeline once one complete vector is available. // Each pipeline preloads its final inputs when a tail overlaps so same-start in-place operation remains correct. if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512.IsSupported - && length >= vector512Threshold) + && length >= (uint)Vector512.Count) { InvokeVectorized512(ref xRef, ref yRef, ref destinationRef, length); return; @@ -255,12 +177,11 @@ internal static partial class TensorPrimitives_ ref T destinationRef = ref MemoryMarshal.GetReference(destination); nuint length = (uint)x.Length; - // The runtime-style unrolled 512-bit pipeline wins on large inputs, but its setup cost regresses the - // shorter JPEG and ICC buffers. Measurements put the crossover safely below 512 elements. + // Runtime main selects the widest supported pipeline once one complete vector is available. if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512.IsSupported - && length >= 512) + && length >= (uint)Vector512.Count) { InvokeVectorized512(ref xRef, y, ref destinationRef, length); return; @@ -285,7 +206,7 @@ internal static partial class TensorPrimitives_ } /// - /// Performs element-wise division using thresholds measured for ImageSharp normalization workloads. + /// Performs element-wise division using the runtime tensor width-selection order. /// /// The element type. /// The division operation to apply. @@ -303,13 +224,11 @@ internal static partial class TensorPrimitives_ ref T destinationRef = ref MemoryMarshal.GetReference(destination); nuint length = (uint)x.Length; - // AVX-512 only wins once there is enough work to amortize its wider dispatch and division latency. - // Eight vectors is also the runtime pipeline's unrolled-loop boundary, while shorter inputs retain - // the lower setup cost of 256-bit vectors. + // Runtime main selects the widest supported pipeline once one complete vector is available. if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512.IsSupported - && length >= (uint)(Vector512.Count * 8)) + && length >= (uint)Vector512.Count) { InvokeVectorized512(ref xRef, y, ref destinationRef, length); return; @@ -726,116 +645,6 @@ internal static partial class TensorPrimitives_ } } - /// - /// Selects maximum single-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 MaxSingle(Vector128 x, Vector128 y) - { - // The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a - // first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order. - Vector128 result = Vector128.Max(x, y); - result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); - - return Vector128.ConditionalSelect( - Vector128.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum single-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 MaxSingle(Vector256 x, Vector256 y) - { - Vector256 result = Vector256.Max(x, y); - result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); - - return Vector256.ConditionalSelect( - Vector256.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum single-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 MaxSingle(Vector512 x, Vector512 y) - { - Vector512 result = Vector512.Max(x, y); - result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); - - return Vector512.ConditionalSelect( - Vector512.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum double-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 MaxDouble(Vector128 x, Vector128 y) - { - Vector128 result = Vector128.Max(x, y); - result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); - - return Vector128.ConditionalSelect( - Vector128.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum double-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 MaxDouble(Vector256 x, Vector256 y) - { - Vector256 result = Vector256.Max(x, y); - result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); - - return Vector256.ConditionalSelect( - Vector256.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum double-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 MaxDouble(Vector512 x, Vector512 y) - { - Vector512 result = Vector512.Max(x, y); - result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); - - return Vector512.ConditionalSelect( - Vector512.Equals(x, y), - x & y, - result); - } - /// /// Applies a ternary operation with 128-bit vectors. /// @@ -1012,567 +821,4 @@ internal static partial class TensorPrimitives_ end.StoreUnsafe(ref destinationRef, length - vectorCount); } } - - /// - /// Clamps single-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 ClampSingle( - Vector128 value, - Vector128 min, - Vector128 max) - { - // Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the - // first operand and select negative zero when equal values have different signs. - Vector128 maximum = Vector128.ConditionalSelect( - Vector128.LessThan(min, value) - | ~Vector128.Equals(value, value) - | (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), - value, - min); - - return Vector128.ConditionalSelect( - Vector128.LessThan(maximum, max) - | ~Vector128.Equals(maximum, maximum) - | (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), - maximum, - max); - } - - /// - /// Clamps single-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 ClampSingle( - Vector256 value, - Vector256 min, - Vector256 max) - { - Vector256 maximum = Vector256.ConditionalSelect( - Vector256.LessThan(min, value) - | ~Vector256.Equals(value, value) - | (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), - value, - min); - - return Vector256.ConditionalSelect( - Vector256.LessThan(maximum, max) - | ~Vector256.Equals(maximum, maximum) - | (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), - maximum, - max); - } - - /// - /// Clamps single-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 ClampSingle( - Vector512 value, - Vector512 min, - Vector512 max) - { - Vector512 maximum = Vector512.ConditionalSelect( - Vector512.LessThan(min, value) - | ~Vector512.Equals(value, value) - | (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), - value, - min); - - return Vector512.ConditionalSelect( - Vector512.LessThan(maximum, max) - | ~Vector512.Equals(maximum, maximum) - | (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), - maximum, - max); - } - - /// - /// Clamps double-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 ClampDouble( - Vector128 value, - Vector128 min, - Vector128 max) - { - Vector128 maximum = Vector128.ConditionalSelect( - Vector128.LessThan(min, value) - | ~Vector128.Equals(value, value) - | (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), - value, - min); - - return Vector128.ConditionalSelect( - Vector128.LessThan(maximum, max) - | ~Vector128.Equals(maximum, maximum) - | (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), - maximum, - max); - } - - /// - /// Clamps double-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 ClampDouble( - Vector256 value, - Vector256 min, - Vector256 max) - { - Vector256 maximum = Vector256.ConditionalSelect( - Vector256.LessThan(min, value) - | ~Vector256.Equals(value, value) - | (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), - value, - min); - - return Vector256.ConditionalSelect( - Vector256.LessThan(maximum, max) - | ~Vector256.Equals(maximum, maximum) - | (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), - maximum, - max); - } - - /// - /// Clamps double-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 ClampDouble( - Vector512 value, - Vector512 min, - Vector512 max) - { - Vector512 maximum = Vector512.ConditionalSelect( - Vector512.LessThan(min, value) - | ~Vector512.Equals(value, value) - | (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), - value, - min); - - return Vector512.ConditionalSelect( - Vector512.LessThan(maximum, max) - | ~Vector512.Equals(maximum, maximum) - | (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), - maximum, - max); - } - - /// - /// Determines whether has the same vector division support as . - /// - /// The element type. - /// when is a 32-bit signed native integer type. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsInt32Like() - => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); - - /// - /// Adds corresponding values. - /// - /// The element type. - private readonly struct AddOperator : IBinaryOperator - where T : IAdditionOperators, IAdditiveIdentity - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Adds scalar values. - /// - /// The first addend. - /// The second addend. - /// The sum. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => x + y; - - /// - /// Adds 128-bit vectors. - /// - /// The first addends. - /// The second addends. - /// The sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y; - - /// - /// Adds 256-bit vectors. - /// - /// The first addends. - /// The second addends. - /// The sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y; - - /// - /// Adds 512-bit vectors. - /// - /// The first addends. - /// The second addends. - /// The sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y; - } - - /// - /// Clamps values using the complete runtime tensor contract, including signed-zero correction. - /// - /// The element type. - private readonly struct ClampOperator : ITernaryOperator - where T : INumber - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Clamps a scalar value. - /// - /// The value. - /// The inclusive lower bound. - /// The inclusive upper bound. - /// The clamped value. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T min, T max) - => Vector128.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max); - - /// - /// Clamps a 128-bit vector. - /// - /// The values. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 min, Vector128 max) - { - if (typeof(T) == typeof(float)) - { - Vector128 result = ClampSingle( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref min), - Unsafe.As, Vector128>(ref max)); - - return Unsafe.As, Vector128>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector128 result = ClampDouble( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref min), - Unsafe.As, Vector128>(ref max)); - - return Unsafe.As, Vector128>(ref result); - } - - return Vector128_.Clamp(x, min, max); - } - - /// - /// Clamps a 256-bit vector. - /// - /// The values. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 min, Vector256 max) - { - if (typeof(T) == typeof(float)) - { - Vector256 result = ClampSingle( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref min), - Unsafe.As, Vector256>(ref max)); - - return Unsafe.As, Vector256>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector256 result = ClampDouble( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref min), - Unsafe.As, Vector256>(ref max)); - - return Unsafe.As, Vector256>(ref result); - } - - return Vector256_.Clamp(x, min, max); - } - - /// - /// Clamps a 512-bit vector. - /// - /// The values. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 min, Vector512 max) - { - if (typeof(T) == typeof(float)) - { - Vector512 result = ClampSingle( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref min), - Unsafe.As, Vector512>(ref max)); - - return Unsafe.As, Vector512>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector512 result = ClampDouble( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref min), - Unsafe.As, Vector512>(ref max)); - - return Unsafe.As, Vector512>(ref result); - } - - return Vector512_.Clamp(x, min, max); - } - } - - /// - /// Selects the maximum corresponding values. - /// - /// The element type. - private readonly struct MaxOperator : IBinaryOperator - where T : INumber - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Selects the maximum scalar value. - /// - /// The first value. - /// The second value. - /// The maximum value. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.Max(x, y); - - /// - /// Selects the maximum values from 128-bit vectors. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (typeof(T) == typeof(float)) - { - Vector128 result = MaxSingle( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref y)); - - return Unsafe.As, Vector128>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector128 result = MaxDouble( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref y)); - - return Unsafe.As, Vector128>(ref result); - } - - return Vector128.Max(x, y); - } - - /// - /// Selects the maximum values from 256-bit vectors. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float)) - { - Vector256 result = MaxSingle( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref y)); - - return Unsafe.As, Vector256>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector256 result = MaxDouble( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref y)); - - return Unsafe.As, Vector256>(ref result); - } - - return Vector256.Max(x, y); - } - - /// - /// Selects the maximum values from 512-bit vectors. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float)) - { - Vector512 result = MaxSingle( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref y)); - - return Unsafe.As, Vector512>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector512 result = MaxDouble( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref y)); - - return Unsafe.As, Vector512>(ref result); - } - - return Vector512.Max(x, y); - } - } - - /// - /// Multiplies corresponding values. - /// - /// The element type. - private readonly struct MultiplyOperator : IBinaryOperator - where T : IMultiplyOperators, IMultiplicativeIdentity - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Multiplies scalar values. - /// - /// The multiplicand. - /// The multiplier. - /// The product. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => x * y; - - /// - /// Multiplies 128-bit vectors. - /// - /// The multiplicands. - /// The multipliers. - /// The products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) => x * y; - - /// - /// Multiplies 256-bit vectors. - /// - /// The multiplicands. - /// The multipliers. - /// The products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) => x * y; - - /// - /// Multiplies 512-bit vectors. - /// - /// The multiplicands. - /// The multipliers. - /// The products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; - } - - /// - /// Divides values by a scalar. - /// - /// The element type. - private readonly struct DivideOperator : IBinaryOperator - where T : IDivisionOperators - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => typeof(T) == typeof(float) - || typeof(T) == typeof(double) - || (Vector256.IsHardwareAccelerated && IsInt32Like()); - - /// - /// Divides scalar values. - /// - /// The dividend. - /// The divisor. - /// The quotient. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => x / y; - - /// - /// Divides 128-bit vectors. - /// - /// The dividends. - /// The divisors. - /// The quotients. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; - - /// - /// Divides 256-bit vectors. - /// - /// The dividends. - /// The divisors. - /// The quotients. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; - - /// - /// Divides 512-bit vectors. - /// - /// The dividends. - /// The divisors. - /// The quotients. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; - } } diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs new file mode 100644 index 000000000..31312dddf --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs @@ -0,0 +1,245 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +internal static partial class TensorPrimitives_ +{ + /// + /// Computes the element-wise maximum of the values in and . + /// + /// The element type. + /// The values to compare. + /// The value to compare with each element. + /// The destination for the maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Max(ReadOnlySpan x, T y, Span destination) + where T : INumber + => InvokeSpanScalarIntoSpan>(x, y, destination); + + /// + /// Selects maximum single-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MaxSingle(Vector128 x, Vector128 y) + { + // The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a + // first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order. + Vector128 result = Vector128.Max(x, y); + result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); + + return Vector128.ConditionalSelect( + Vector128.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum single-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MaxSingle(Vector256 x, Vector256 y) + { + Vector256 result = Vector256.Max(x, y); + result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); + + return Vector256.ConditionalSelect( + Vector256.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum single-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 MaxSingle(Vector512 x, Vector512 y) + { + Vector512 result = Vector512.Max(x, y); + result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); + + return Vector512.ConditionalSelect( + Vector512.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum double-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MaxDouble(Vector128 x, Vector128 y) + { + Vector128 result = Vector128.Max(x, y); + result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); + + return Vector128.ConditionalSelect( + Vector128.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum double-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MaxDouble(Vector256 x, Vector256 y) + { + Vector256 result = Vector256.Max(x, y); + result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); + + return Vector256.ConditionalSelect( + Vector256.Equals(x, y), + x & y, + result); + } + + /// + /// Selects maximum double-precision values with the normalized runtime semantics. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 MaxDouble(Vector512 x, Vector512 y) + { + Vector512 result = Vector512.Max(x, y); + result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); + + return Vector512.ConditionalSelect( + Vector512.Equals(x, y), + x & y, + result); + } + + /// + /// Selects the maximum corresponding values. + /// + /// The element type. + private readonly struct MaxOperator : IBinaryOperator + where T : INumber + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Selects the maximum scalar value. + /// + /// The first value. + /// The second value. + /// The maximum value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.Max(x, y); + + /// + /// Selects the maximum values from 128-bit vectors. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (typeof(T) == typeof(float)) + { + Vector128 result = MaxSingle( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref y)); + + return Unsafe.As, Vector128>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector128 result = MaxDouble( + Unsafe.As, Vector128>(ref x), + Unsafe.As, Vector128>(ref y)); + + return Unsafe.As, Vector128>(ref result); + } + + return Vector128.Max(x, y); + } + + /// + /// Selects the maximum values from 256-bit vectors. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float)) + { + Vector256 result = MaxSingle( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref y)); + + return Unsafe.As, Vector256>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector256 result = MaxDouble( + Unsafe.As, Vector256>(ref x), + Unsafe.As, Vector256>(ref y)); + + return Unsafe.As, Vector256>(ref result); + } + + return Vector256.Max(x, y); + } + + /// + /// Selects the maximum values from 512-bit vectors. + /// + /// The first values. + /// The second values. + /// The maximum values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float)) + { + Vector512 result = MaxSingle( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref y)); + + return Unsafe.As, Vector512>(ref result); + } + + if (typeof(T) == typeof(double)) + { + Vector512 result = MaxDouble( + Unsafe.As, Vector512>(ref x), + Unsafe.As, Vector512>(ref y)); + + return Unsafe.As, Vector512>(ref result); + } + + return Vector512.Max(x, y); + } + } +} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs new file mode 100644 index 000000000..deb1e922e --- /dev/null +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs @@ -0,0 +1,72 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Common.Helpers; + +internal static partial class TensorPrimitives_ +{ + /// + /// Computes the element-wise product of the values in and . + /// + /// The element type. + /// The multiplicands. + /// The multiplier. + /// The destination for the products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Multiply(ReadOnlySpan x, T y, Span destination) + where T : IMultiplyOperators, IMultiplicativeIdentity + => InvokeSpanScalarIntoSpan>(x, y, destination); + + /// + /// Multiplies corresponding values. + /// + /// The element type. + private readonly struct MultiplyOperator : IBinaryOperator + where T : IMultiplyOperators, IMultiplicativeIdentity + { + /// + /// Gets a value indicating whether this operation supports vector execution. + /// + public static bool Vectorizable => true; + + /// + /// Multiplies scalar values. + /// + /// The multiplicand. + /// The multiplier. + /// The product. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => x * y; + + /// + /// Multiplies 128-bit vectors. + /// + /// The multiplicands. + /// The multipliers. + /// The products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => x * y; + + /// + /// Multiplies 256-bit vectors. + /// + /// The multiplicands. + /// The multipliers. + /// The products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => x * y; + + /// + /// Multiplies 512-bit vectors. + /// + /// The multiplicands. + /// The multipliers. + /// The products. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; + } +} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs index f61e4877c..eb90a3903 100644 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs +++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs @@ -76,12 +76,11 @@ internal static partial class TensorPrimitives_ ref T destinationRef = ref MemoryMarshal.GetReference(destination); nuint length = (uint)x.Length; - // The dispatch matches the other compatibility pipelines: AVX-512 is reserved for large spans because - // its setup cost is not recovered by the short image-processing buffers that dominate ImageSharp. + // Runtime main selects the widest supported pipeline once one complete vector is available. if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512.IsSupported - && length >= 512) + && length >= (uint)Vector512.Count) { InvokeUnaryVectorized512(ref xRef, ref destinationRef, length); return;