diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs
new file mode 100644
index 000000000..f2ef214cf
--- /dev/null
+++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs
@@ -0,0 +1,84 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Common.Helpers;
+
+internal static partial class TensorPrimitives_
+{
+ ///
+ /// Computes the element-wise sum of the values in and .
+ ///
+ /// The element type.
+ /// The first addends.
+ /// The second addends.
+ /// The destination for the sums.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination)
+ where T : IAdditionOperators, IAdditiveIdentity
+ => InvokeSpanSpanIntoSpan>(x, y, destination);
+
+ ///
+ /// Computes the element-wise sum of the values in and the scalar .
+ ///
+ /// The element type.
+ /// The first addends.
+ /// The scalar second addend.
+ /// The destination for the sums.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void Add(ReadOnlySpan x, T y, Span destination)
+ where T : IAdditionOperators, IAdditiveIdentity
+ => InvokeSpanScalarIntoSpan>(x, y, destination);
+
+ ///
+ /// Adds corresponding values.
+ ///
+ /// The element type.
+ private readonly struct AddOperator : IBinaryOperator
+ where T : IAdditionOperators, IAdditiveIdentity
+ {
+ ///
+ /// Gets a value indicating whether this operation supports vector execution.
+ ///
+ public static bool Vectorizable => true;
+
+ ///
+ /// Adds scalar values.
+ ///
+ /// The first addend.
+ /// The second addend.
+ /// The sum.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static T Invoke(T x, T y) => x + y;
+
+ ///
+ /// Adds 128-bit vectors.
+ ///
+ /// The first addends.
+ /// The second addends.
+ /// The sums.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y;
+
+ ///
+ /// Adds 256-bit vectors.
+ ///
+ /// The first addends.
+ /// The second addends.
+ /// The sums.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y;
+
+ ///
+ /// Adds 512-bit vectors.
+ ///
+ /// The first addends.
+ /// The second addends.
+ /// The sums.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y;
+ }
+}
diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs
new file mode 100644
index 000000000..e2cc9097e
--- /dev/null
+++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs
@@ -0,0 +1,318 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Common.Helpers;
+
+internal static partial class TensorPrimitives_
+{
+ ///
+ /// Computes the element-wise result of clamping to the inclusive range specified
+ /// by and .
+ ///
+ /// The element type.
+ /// The values to clamp.
+ /// The inclusive lower bound.
+ /// The inclusive upper bound.
+ /// The destination for the clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void Clamp(ReadOnlySpan x, T min, T max, Span destination)
+ where T : INumber
+ => InvokeSpanScalarScalarIntoSpan>(x, min, max, destination);
+
+ ///
+ /// Clamps single-precision values with the normalized runtime semantics.
+ ///
+ /// The values to clamp.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 ClampSingle(
+ Vector128 value,
+ Vector128 min,
+ Vector128 max)
+ {
+ // Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the
+ // first operand and select negative zero when equal values have different signs.
+ Vector128 maximum = Vector128.ConditionalSelect(
+ Vector128.LessThan(min, value)
+ | ~Vector128.Equals(value, value)
+ | (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()),
+ value,
+ min);
+
+ return Vector128.ConditionalSelect(
+ Vector128.LessThan(maximum, max)
+ | ~Vector128.Equals(maximum, maximum)
+ | (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()),
+ maximum,
+ max);
+ }
+
+ ///
+ /// Clamps single-precision values with the normalized runtime semantics.
+ ///
+ /// The values to clamp.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector256 ClampSingle(
+ Vector256 value,
+ Vector256 min,
+ Vector256 max)
+ {
+ Vector256 maximum = Vector256.ConditionalSelect(
+ Vector256.LessThan(min, value)
+ | ~Vector256.Equals(value, value)
+ | (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()),
+ value,
+ min);
+
+ return Vector256.ConditionalSelect(
+ Vector256.LessThan(maximum, max)
+ | ~Vector256.Equals(maximum, maximum)
+ | (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()),
+ maximum,
+ max);
+ }
+
+ ///
+ /// Clamps single-precision values with the normalized runtime semantics.
+ ///
+ /// The values to clamp.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 ClampSingle(
+ Vector512 value,
+ Vector512 min,
+ Vector512 max)
+ {
+ Vector512 maximum = Vector512.ConditionalSelect(
+ Vector512.LessThan(min, value)
+ | ~Vector512.Equals(value, value)
+ | (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()),
+ value,
+ min);
+
+ return Vector512.ConditionalSelect(
+ Vector512.LessThan(maximum, max)
+ | ~Vector512.Equals(maximum, maximum)
+ | (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()),
+ maximum,
+ max);
+ }
+
+ ///
+ /// Clamps double-precision values with the normalized runtime semantics.
+ ///
+ /// The values to clamp.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 ClampDouble(
+ Vector128 value,
+ Vector128 min,
+ Vector128 max)
+ {
+ Vector128 maximum = Vector128.ConditionalSelect(
+ Vector128.LessThan(min, value)
+ | ~Vector128.Equals(value, value)
+ | (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()),
+ value,
+ min);
+
+ return Vector128.ConditionalSelect(
+ Vector128.LessThan(maximum, max)
+ | ~Vector128.Equals(maximum, maximum)
+ | (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()),
+ maximum,
+ max);
+ }
+
+ ///
+ /// Clamps double-precision values with the normalized runtime semantics.
+ ///
+ /// The values to clamp.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector256 ClampDouble(
+ Vector256 value,
+ Vector256 min,
+ Vector256 max)
+ {
+ Vector256 maximum = Vector256.ConditionalSelect(
+ Vector256.LessThan(min, value)
+ | ~Vector256.Equals(value, value)
+ | (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()),
+ value,
+ min);
+
+ return Vector256.ConditionalSelect(
+ Vector256.LessThan(maximum, max)
+ | ~Vector256.Equals(maximum, maximum)
+ | (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()),
+ maximum,
+ max);
+ }
+
+ ///
+ /// Clamps double-precision values with the normalized runtime semantics.
+ ///
+ /// The values to clamp.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 ClampDouble(
+ Vector512 value,
+ Vector512 min,
+ Vector512 max)
+ {
+ Vector512 maximum = Vector512.ConditionalSelect(
+ Vector512.LessThan(min, value)
+ | ~Vector512.Equals(value, value)
+ | (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()),
+ value,
+ min);
+
+ return Vector512.ConditionalSelect(
+ Vector512.LessThan(maximum, max)
+ | ~Vector512.Equals(maximum, maximum)
+ | (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()),
+ maximum,
+ max);
+ }
+
+ ///
+ /// Clamps values using the complete runtime tensor contract, including signed-zero correction.
+ ///
+ /// The element type.
+ private readonly struct ClampOperator : ITernaryOperator
+ where T : INumber
+ {
+ ///
+ /// Gets a value indicating whether this operation supports vector execution.
+ ///
+ public static bool Vectorizable => true;
+
+ ///
+ /// Clamps a scalar value.
+ ///
+ /// The value.
+ /// The inclusive lower bound.
+ /// The inclusive upper bound.
+ /// The clamped value.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static T Invoke(T x, T min, T max)
+ => Vector128.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max);
+
+ ///
+ /// Clamps a 128-bit vector.
+ ///
+ /// The values.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 x, Vector128 min, Vector128 max)
+ {
+ if (typeof(T) == typeof(float))
+ {
+ Vector128 result = ClampSingle(
+ Unsafe.As, Vector128>(ref x),
+ Unsafe.As, Vector128>(ref min),
+ Unsafe.As, Vector128>(ref max));
+
+ return Unsafe.As, Vector128>(ref result);
+ }
+
+ if (typeof(T) == typeof(double))
+ {
+ Vector128 result = ClampDouble(
+ Unsafe.As, Vector128>(ref x),
+ Unsafe.As, Vector128>(ref min),
+ Unsafe.As, Vector128>(ref max));
+
+ return Unsafe.As, Vector128>(ref result);
+ }
+
+ return Vector128_.Clamp(x, min, max);
+ }
+
+ ///
+ /// Clamps a 256-bit vector.
+ ///
+ /// The values.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 x, Vector256 min, Vector256 max)
+ {
+ if (typeof(T) == typeof(float))
+ {
+ Vector256 result = ClampSingle(
+ Unsafe.As, Vector256>(ref x),
+ Unsafe.As, Vector256>(ref min),
+ Unsafe.As, Vector256>(ref max));
+
+ return Unsafe.As, Vector256>(ref result);
+ }
+
+ if (typeof(T) == typeof(double))
+ {
+ Vector256 result = ClampDouble(
+ Unsafe.As, Vector256>(ref x),
+ Unsafe.As, Vector256>(ref min),
+ Unsafe.As, Vector256>(ref max));
+
+ return Unsafe.As, Vector256>(ref result);
+ }
+
+ return Vector256_.Clamp(x, min, max);
+ }
+
+ ///
+ /// Clamps a 512-bit vector.
+ ///
+ /// The values.
+ /// The inclusive lower bounds.
+ /// The inclusive upper bounds.
+ /// The clamped values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 x, Vector512 min, Vector512 max)
+ {
+ if (typeof(T) == typeof(float))
+ {
+ Vector512 result = ClampSingle(
+ Unsafe.As, Vector512>(ref x),
+ Unsafe.As, Vector512>(ref min),
+ Unsafe.As, Vector512>(ref max));
+
+ return Unsafe.As, Vector512>(ref result);
+ }
+
+ if (typeof(T) == typeof(double))
+ {
+ Vector512 result = ClampDouble(
+ Unsafe.As, Vector512>(ref x),
+ Unsafe.As, Vector512>(ref min),
+ Unsafe.As, Vector512>(ref max));
+
+ return Unsafe.As, Vector512>(ref result);
+ }
+
+ return Vector512_.Clamp(x, min, max);
+ }
+ }
+}
diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs
new file mode 100644
index 000000000..e40a26cea
--- /dev/null
+++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs
@@ -0,0 +1,83 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+namespace SixLabors.ImageSharp.Common.Helpers;
+
+internal static partial class TensorPrimitives_
+{
+ ///
+ /// Computes the element-wise result of dividing the values in by .
+ ///
+ /// The element type.
+ /// The dividend values.
+ /// The divisor.
+ /// The destination for the quotient values.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void Divide(ReadOnlySpan x, T y, Span destination)
+ where T : IDivisionOperators
+ => InvokeSpanScalarIntoSpanForDivision>(x, y, destination);
+
+ ///
+ /// Determines whether has the same vector division support as .
+ ///
+ /// The element type.
+ /// when is a 32-bit signed native integer type.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static bool IsInt32Like()
+ => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint));
+
+ ///
+ /// Divides values by a scalar.
+ ///
+ /// The element type.
+ private readonly struct DivideOperator : IBinaryOperator
+ where T : IDivisionOperators
+ {
+ ///
+ /// Gets a value indicating whether this operation supports vector execution.
+ ///
+ public static bool Vectorizable => typeof(T) == typeof(float)
+ || typeof(T) == typeof(double)
+ || (Vector256.IsHardwareAccelerated && IsInt32Like());
+
+ ///
+ /// Divides scalar values.
+ ///
+ /// The dividend.
+ /// The divisor.
+ /// The quotient.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static T Invoke(T x, T y) => x / y;
+
+ ///
+ /// Divides 128-bit vectors.
+ ///
+ /// The dividends.
+ /// The divisors.
+ /// The quotients.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y;
+
+ ///
+ /// Divides 256-bit vectors.
+ ///
+ /// The dividends.
+ /// The divisors.
+ /// The quotients.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y;
+
+ ///
+ /// Divides 512-bit vectors.
+ ///
+ /// The dividends.
+ /// The divisors.
+ /// The quotients.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y;
+ }
+}
diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs
similarity index 55%
rename from src/ImageSharp/Common/Helpers/TensorPrimitives.cs
rename to src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs
index 4be4ab3aa..3ba1b61af 100644
--- a/src/ImageSharp/Common/Helpers/TensorPrimitives.cs
+++ b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs
@@ -15,9 +15,7 @@ namespace SixLabors.ImageSharp.Common.Helpers;
/// The API shape follows System.Numerics.Tensors.TensorPrimitives so call sites can move to the runtime
/// implementation when ImageSharp no longer supports target frameworks that predate it.
///
-#pragma warning disable SA1649 // File name should match first type name
internal static partial class TensorPrimitives_
-#pragma warning restore SA1649 // File name should match first type name
{
///
/// Defines an element-wise binary operation.
@@ -111,80 +109,6 @@ internal static partial class TensorPrimitives_
public static abstract Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z);
}
- ///
- /// Computes the element-wise result of clamping to the inclusive range specified
- /// by and .
- ///
- /// The element type.
- /// The values to clamp.
- /// The inclusive lower bound.
- /// The inclusive upper bound.
- /// The destination for the clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Clamp(ReadOnlySpan x, T min, T max, Span destination)
- where T : INumber
- => InvokeSpanScalarScalarIntoSpan>(x, min, max, destination);
-
- ///
- /// Computes the element-wise sum of the values in and .
- ///
- /// The element type.
- /// The first addends.
- /// The second addends.
- /// The destination for the sums.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination)
- where T : IAdditionOperators, IAdditiveIdentity
- => InvokeSpanSpanIntoSpan>(x, y, destination);
-
- ///
- /// Computes the element-wise sum of the values in and the scalar .
- ///
- /// The element type.
- /// The first addends.
- /// The scalar second addend.
- /// The destination for the sums.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Add(ReadOnlySpan x, T y, Span destination)
- where T : IAdditionOperators, IAdditiveIdentity
- => InvokeSpanScalarIntoSpan>(x, y, destination);
-
- ///
- /// Computes the element-wise result of dividing the values in by .
- ///
- /// The element type.
- /// The dividend values.
- /// The divisor.
- /// The destination for the quotient values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Divide(ReadOnlySpan x, T y, Span destination)
- where T : IDivisionOperators
- => InvokeSpanScalarIntoSpanForDivision>(x, y, destination);
-
- ///
- /// Computes the element-wise maximum of the values in and .
- ///
- /// The element type.
- /// The values to compare.
- /// The value to compare with each element.
- /// The destination for the maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Max(ReadOnlySpan x, T y, Span destination)
- where T : INumber
- => InvokeSpanScalarIntoSpan>(x, y, destination);
-
- ///
- /// Computes the element-wise product of the values in and .
- ///
- /// The element type.
- /// The multiplicands.
- /// The multiplier.
- /// The destination for the products.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Multiply(ReadOnlySpan x, T y, Span destination)
- where T : IMultiplyOperators, IMultiplicativeIdentity
- => InvokeSpanScalarIntoSpan>(x, y, destination);
-
///
/// Performs an element-wise binary operation between two spans.
///
@@ -204,15 +128,13 @@ internal static partial class TensorPrimitives_
ref T yRef = ref MemoryMarshal.GetReference(y);
ref T destinationRef = ref MemoryMarshal.GetReference(destination);
nuint length = (uint)x.Length;
- nuint vector512Threshold = Unsafe.SizeOf() == 1 ? (uint)Vector512.Count : 512;
- // Match the runtime's AVX-512 selection for byte-sized elements once one complete vector is available.
- // Wider elements retain the measured crossover point where their 512-bit setup cost becomes worthwhile.
+ // Runtime main selects the widest supported pipeline once one complete vector is available.
// Each pipeline preloads its final inputs when a tail overlaps so same-start in-place operation remains correct.
if (TOperator.Vectorizable
&& Vector512.IsHardwareAccelerated
&& Vector512.IsSupported
- && length >= vector512Threshold)
+ && length >= (uint)Vector512.Count)
{
InvokeVectorized512(ref xRef, ref yRef, ref destinationRef, length);
return;
@@ -255,12 +177,11 @@ internal static partial class TensorPrimitives_
ref T destinationRef = ref MemoryMarshal.GetReference(destination);
nuint length = (uint)x.Length;
- // The runtime-style unrolled 512-bit pipeline wins on large inputs, but its setup cost regresses the
- // shorter JPEG and ICC buffers. Measurements put the crossover safely below 512 elements.
+ // Runtime main selects the widest supported pipeline once one complete vector is available.
if (TOperator.Vectorizable
&& Vector512.IsHardwareAccelerated
&& Vector512.IsSupported
- && length >= 512)
+ && length >= (uint)Vector512.Count)
{
InvokeVectorized512(ref xRef, y, ref destinationRef, length);
return;
@@ -285,7 +206,7 @@ internal static partial class TensorPrimitives_
}
///
- /// Performs element-wise division using thresholds measured for ImageSharp normalization workloads.
+ /// Performs element-wise division using the runtime tensor width-selection order.
///
/// The element type.
/// The division operation to apply.
@@ -303,13 +224,11 @@ internal static partial class TensorPrimitives_
ref T destinationRef = ref MemoryMarshal.GetReference(destination);
nuint length = (uint)x.Length;
- // AVX-512 only wins once there is enough work to amortize its wider dispatch and division latency.
- // Eight vectors is also the runtime pipeline's unrolled-loop boundary, while shorter inputs retain
- // the lower setup cost of 256-bit vectors.
+ // Runtime main selects the widest supported pipeline once one complete vector is available.
if (TOperator.Vectorizable
&& Vector512.IsHardwareAccelerated
&& Vector512.IsSupported
- && length >= (uint)(Vector512.Count * 8))
+ && length >= (uint)Vector512.Count)
{
InvokeVectorized512(ref xRef, y, ref destinationRef, length);
return;
@@ -726,116 +645,6 @@ internal static partial class TensorPrimitives_
}
}
- ///
- /// Selects maximum single-precision values with the normalized runtime semantics.
- ///
- /// The first values.
- /// The second values.
- /// The maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector128 MaxSingle(Vector128 x, Vector128 y)
- {
- // The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a
- // first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order.
- Vector128 result = Vector128.Max(x, y);
- result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result);
-
- return Vector128.ConditionalSelect(
- Vector128.Equals(x, y),
- x & y,
- result);
- }
-
- ///
- /// Selects maximum single-precision values with the normalized runtime semantics.
- ///
- /// The first values.
- /// The second values.
- /// The maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector256 MaxSingle(Vector256 x, Vector256 y)
- {
- Vector256 result = Vector256.Max(x, y);
- result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result);
-
- return Vector256.ConditionalSelect(
- Vector256.Equals(x, y),
- x & y,
- result);
- }
-
- ///
- /// Selects maximum single-precision values with the normalized runtime semantics.
- ///
- /// The first values.
- /// The second values.
- /// The maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector512 MaxSingle(Vector512 x, Vector512 y)
- {
- Vector512 result = Vector512.Max(x, y);
- result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result);
-
- return Vector512.ConditionalSelect(
- Vector512.Equals(x, y),
- x & y,
- result);
- }
-
- ///
- /// Selects maximum double-precision values with the normalized runtime semantics.
- ///
- /// The first values.
- /// The second values.
- /// The maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector128 MaxDouble(Vector128 x, Vector128 y)
- {
- Vector128 result = Vector128.Max(x, y);
- result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result);
-
- return Vector128.ConditionalSelect(
- Vector128.Equals(x, y),
- x & y,
- result);
- }
-
- ///
- /// Selects maximum double-precision values with the normalized runtime semantics.
- ///
- /// The first values.
- /// The second values.
- /// The maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector256 MaxDouble(Vector256 x, Vector256 y)
- {
- Vector256 result = Vector256.Max(x, y);
- result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result);
-
- return Vector256.ConditionalSelect(
- Vector256.Equals(x, y),
- x & y,
- result);
- }
-
- ///
- /// Selects maximum double-precision values with the normalized runtime semantics.
- ///
- /// The first values.
- /// The second values.
- /// The maximum values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector512 MaxDouble(Vector512 x, Vector512 y)
- {
- Vector512 result = Vector512.Max(x, y);
- result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result);
-
- return Vector512.ConditionalSelect(
- Vector512.Equals(x, y),
- x & y,
- result);
- }
-
///
/// Applies a ternary operation with 128-bit vectors.
///
@@ -1012,567 +821,4 @@ internal static partial class TensorPrimitives_
end.StoreUnsafe(ref destinationRef, length - vectorCount);
}
}
-
- ///
- /// Clamps single-precision values with the normalized runtime semantics.
- ///
- /// The values to clamp.
- /// The inclusive lower bounds.
- /// The inclusive upper bounds.
- /// The clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector128 ClampSingle(
- Vector128 value,
- Vector128 min,
- Vector128 max)
- {
- // Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the
- // first operand and select negative zero when equal values have different signs.
- Vector128 maximum = Vector128.ConditionalSelect(
- Vector128.LessThan(min, value)
- | ~Vector128.Equals(value, value)
- | (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()),
- value,
- min);
-
- return Vector128.ConditionalSelect(
- Vector128.LessThan(maximum, max)
- | ~Vector128.Equals(maximum, maximum)
- | (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()),
- maximum,
- max);
- }
-
- ///
- /// Clamps single-precision values with the normalized runtime semantics.
- ///
- /// The values to clamp.
- /// The inclusive lower bounds.
- /// The inclusive upper bounds.
- /// The clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector256 ClampSingle(
- Vector256 value,
- Vector256 min,
- Vector256 max)
- {
- Vector256 maximum = Vector256.ConditionalSelect(
- Vector256.LessThan(min, value)
- | ~Vector256.Equals(value, value)
- | (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()),
- value,
- min);
-
- return Vector256.ConditionalSelect(
- Vector256.LessThan(maximum, max)
- | ~Vector256.Equals(maximum, maximum)
- | (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()),
- maximum,
- max);
- }
-
- ///
- /// Clamps single-precision values with the normalized runtime semantics.
- ///
- /// The values to clamp.
- /// The inclusive lower bounds.
- /// The inclusive upper bounds.
- /// The clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector512 ClampSingle(
- Vector512 value,
- Vector512 min,
- Vector512 max)
- {
- Vector512 maximum = Vector512.ConditionalSelect(
- Vector512.LessThan(min, value)
- | ~Vector512.Equals(value, value)
- | (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()),
- value,
- min);
-
- return Vector512.ConditionalSelect(
- Vector512.LessThan(maximum, max)
- | ~Vector512.Equals(maximum, maximum)
- | (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()),
- maximum,
- max);
- }
-
- ///
- /// Clamps double-precision values with the normalized runtime semantics.
- ///
- /// The values to clamp.
- /// The inclusive lower bounds.
- /// The inclusive upper bounds.
- /// The clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector128 ClampDouble(
- Vector128 value,
- Vector128 min,
- Vector128 max)
- {
- Vector128 maximum = Vector128.ConditionalSelect(
- Vector128.LessThan(min, value)
- | ~Vector128.Equals(value, value)
- | (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()),
- value,
- min);
-
- return Vector128.ConditionalSelect(
- Vector128.LessThan(maximum, max)
- | ~Vector128.Equals(maximum, maximum)
- | (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()),
- maximum,
- max);
- }
-
- ///
- /// Clamps double-precision values with the normalized runtime semantics.
- ///
- /// The values to clamp.
- /// The inclusive lower bounds.
- /// The inclusive upper bounds.
- /// The clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector256 ClampDouble(
- Vector256 value,
- Vector256 min,
- Vector256 max)
- {
- Vector256 maximum = Vector256.ConditionalSelect(
- Vector256.LessThan(min, value)
- | ~Vector256.Equals(value, value)
- | (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()),
- value,
- min);
-
- return Vector256.ConditionalSelect(
- Vector256.LessThan(maximum, max)
- | ~Vector256.Equals(maximum, maximum)
- | (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()),
- maximum,
- max);
- }
-
- ///
- /// Clamps double-precision values with the normalized runtime semantics.
- ///
- /// The values to clamp.
- /// The inclusive lower bounds.
- /// The inclusive upper bounds.
- /// The clamped values.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static Vector512 ClampDouble(
- Vector512 value,
- Vector512 min,
- Vector512 max)
- {
- Vector512