mirror of https://github.com/SixLabors/ImageSharp
Browse Source
Replaced ImageSharp’s custom TensorPrimitives_ and several Vector128/256/512 helper fallbacks with direct System.Numerics and System.Numerics.Tensors APIs across color conversion, codecs, filtering, blending, and convolution paths. Removed the obsolete compatibility helper files, updated package references (including System.Numerics.Tensors and newer System.IO.Hashing), and aligned tests with runtime behavior changes such as inverted float clamp bounds now throwing ArgumentException.pull/3179/head
James Jackson-South
2 months ago
50 changed files with 397 additions and 3745 deletions
@ -1,93 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise sum of the values in <paramref name="x"/> and <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <param name="destination">The destination for the sums.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="x"/> and <paramref name="y"/> do not have the same length.</exception>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than the input spans.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// An input and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Add<T>(ReadOnlySpan<T> x, ReadOnlySpan<T> y, Span<T> destination) |
|||
where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T> |
|||
=> InvokeSpanSpanIntoSpan<T, AddOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Computes the element-wise sum of the values in <paramref name="x"/> and the scalar <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The scalar second addend.</param>
|
|||
/// <param name="destination">The destination for the sums.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Add<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T> |
|||
=> InvokeSpanScalarIntoSpan<T, AddOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Adds corresponding values.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct AddOperator<T> : IBinaryOperator<T> |
|||
where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Adds scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addend.</param>
|
|||
/// <param name="y">The second addend.</param>
|
|||
/// <returns>The sum.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => x + y; |
|||
|
|||
/// <summary>
|
|||
/// Adds 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <returns>The sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) => x + y; |
|||
|
|||
/// <summary>
|
|||
/// Adds 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <returns>The sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) => x + y; |
|||
|
|||
/// <summary>
|
|||
/// Adds 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <returns>The sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) => x + y; |
|||
} |
|||
} |
|||
@ -1,322 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise result of clamping <paramref name="x"/> to the inclusive range specified
|
|||
/// by <paramref name="min"/> and <paramref name="max"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bound.</param>
|
|||
/// <param name="max">The inclusive upper bound.</param>
|
|||
/// <param name="destination">The destination for the clamped values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Clamp<T>(ReadOnlySpan<T> x, T min, T max, Span<T> destination) |
|||
where T : INumber<T> |
|||
=> InvokeSpanScalarScalarIntoSpan<T, ClampOperator<T>>(x, min, max, destination); |
|||
|
|||
/// <summary>
|
|||
/// Clamps single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<float> ClampSingle( |
|||
Vector128<float> value, |
|||
Vector128<float> min, |
|||
Vector128<float> max) |
|||
{ |
|||
// Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the
|
|||
// first operand and select negative zero when equal values have different signs.
|
|||
Vector128<float> maximum = Vector128.ConditionalSelect( |
|||
Vector128.LessThan(min, value) |
|||
| ~Vector128.Equals(value, value) |
|||
| (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.LessThan(maximum, max) |
|||
| ~Vector128.Equals(maximum, maximum) |
|||
| (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<float> ClampSingle( |
|||
Vector256<float> value, |
|||
Vector256<float> min, |
|||
Vector256<float> max) |
|||
{ |
|||
Vector256<float> maximum = Vector256.ConditionalSelect( |
|||
Vector256.LessThan(min, value) |
|||
| ~Vector256.Equals(value, value) |
|||
| (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.LessThan(maximum, max) |
|||
| ~Vector256.Equals(maximum, maximum) |
|||
| (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<float> ClampSingle( |
|||
Vector512<float> value, |
|||
Vector512<float> min, |
|||
Vector512<float> max) |
|||
{ |
|||
Vector512<float> maximum = Vector512.ConditionalSelect( |
|||
Vector512.LessThan(min, value) |
|||
| ~Vector512.Equals(value, value) |
|||
| (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.LessThan(maximum, max) |
|||
| ~Vector512.Equals(maximum, maximum) |
|||
| (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<double> ClampDouble( |
|||
Vector128<double> value, |
|||
Vector128<double> min, |
|||
Vector128<double> max) |
|||
{ |
|||
Vector128<double> maximum = Vector128.ConditionalSelect( |
|||
Vector128.LessThan(min, value) |
|||
| ~Vector128.Equals(value, value) |
|||
| (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.LessThan(maximum, max) |
|||
| ~Vector128.Equals(maximum, maximum) |
|||
| (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<double> ClampDouble( |
|||
Vector256<double> value, |
|||
Vector256<double> min, |
|||
Vector256<double> max) |
|||
{ |
|||
Vector256<double> maximum = Vector256.ConditionalSelect( |
|||
Vector256.LessThan(min, value) |
|||
| ~Vector256.Equals(value, value) |
|||
| (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.LessThan(maximum, max) |
|||
| ~Vector256.Equals(maximum, maximum) |
|||
| (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<double> ClampDouble( |
|||
Vector512<double> value, |
|||
Vector512<double> min, |
|||
Vector512<double> max) |
|||
{ |
|||
Vector512<double> maximum = Vector512.ConditionalSelect( |
|||
Vector512.LessThan(min, value) |
|||
| ~Vector512.Equals(value, value) |
|||
| (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.LessThan(maximum, max) |
|||
| ~Vector512.Equals(maximum, maximum) |
|||
| (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps values using the complete runtime tensor contract, including signed-zero correction.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct ClampOperator<T> : ITernaryOperator<T> |
|||
where T : INumber<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Clamps a scalar value.
|
|||
/// </summary>
|
|||
/// <param name="x">The value.</param>
|
|||
/// <param name="min">The inclusive lower bound.</param>
|
|||
/// <param name="max">The inclusive upper bound.</param>
|
|||
/// <returns>The clamped value.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T min, T max) |
|||
=> Vector128<T>.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max); |
|||
|
|||
/// <summary>
|
|||
/// Clamps a 128-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The values.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> min, Vector128<T> max) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector128<float> result = ClampSingle( |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref min), |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector128<float>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector128<double> result = ClampDouble( |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref min), |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector128<double>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
return Vector128_.Clamp(x, min, max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps a 256-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The values.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> min, Vector256<T> max) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector256<float> result = ClampSingle( |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref min), |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector256<float>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector256<double> result = ClampDouble( |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref min), |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector256<double>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
return Vector256_.Clamp(x, min, max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps a 512-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The values.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> min, Vector512<T> max) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector512<float> result = ClampSingle( |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref min), |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector512<float>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector512<double> result = ClampDouble( |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref min), |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector512<double>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
return Vector512_.Clamp(x, min, max); |
|||
} |
|||
} |
|||
} |
|||
@ -1,87 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise result of dividing the values in <paramref name="x"/> by <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The dividend values.</param>
|
|||
/// <param name="y">The divisor.</param>
|
|||
/// <param name="destination">The destination for the quotient values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Divide<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : IDivisionOperators<T, T, T> |
|||
=> InvokeSpanScalarIntoSpanForDivision<T, DivideOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Determines whether <typeparamref name="T"/> has the same vector division support as <see cref="int"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <returns><see langword="true"/> when <typeparamref name="T"/> is a 32-bit signed native integer type.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static bool IsInt32Like<T>() |
|||
=> typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); |
|||
|
|||
/// <summary>
|
|||
/// Divides values by a scalar.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct DivideOperator<T> : IBinaryOperator<T> |
|||
where T : IDivisionOperators<T, T, T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => typeof(T) == typeof(float) |
|||
|| typeof(T) == typeof(double) |
|||
|| (Vector256.IsHardwareAccelerated && IsInt32Like<T>()); |
|||
|
|||
/// <summary>
|
|||
/// Divides scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividend.</param>
|
|||
/// <param name="y">The divisor.</param>
|
|||
/// <returns>The quotient.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => x / y; |
|||
|
|||
/// <summary>
|
|||
/// Divides 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividends.</param>
|
|||
/// <param name="y">The divisors.</param>
|
|||
/// <returns>The quotients.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) => x / y; |
|||
|
|||
/// <summary>
|
|||
/// Divides 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividends.</param>
|
|||
/// <param name="y">The divisors.</param>
|
|||
/// <returns>The quotients.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) => x / y; |
|||
|
|||
/// <summary>
|
|||
/// Divides 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividends.</param>
|
|||
/// <param name="y">The divisors.</param>
|
|||
/// <returns>The quotients.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) => x / y; |
|||
} |
|||
} |
|||
@ -1,900 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Diagnostics.CodeAnalysis; |
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
/// <summary>
|
|||
/// Provides compatibility implementations for tensor operations that are not available on every target framework.
|
|||
/// </summary>
|
|||
/// <remarks>
|
|||
/// The API shape follows <c>System.Numerics.Tensors.TensorPrimitives</c> so call sites can move to the runtime
|
|||
/// implementation when ImageSharp no longer supports target frameworks that predate it.
|
|||
/// </remarks>
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Defines an element-wise binary operation.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private interface IBinaryOperator<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether the operation supports vector execution.
|
|||
/// </summary>
|
|||
public static abstract bool Vectorizable { get; } |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The first value.</param>
|
|||
/// <param name="y">The second value.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract T Invoke(T x, T y); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector128<T> Invoke(Vector128<T> x, Vector128<T> y); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector256<T> Invoke(Vector256<T> x, Vector256<T> y); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector512<T> Invoke(Vector512<T> x, Vector512<T> y); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Defines an element-wise ternary operation.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private interface ITernaryOperator<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether the operation supports vector execution.
|
|||
/// </summary>
|
|||
public static abstract bool Vectorizable { get; } |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The first value.</param>
|
|||
/// <param name="y">The second value.</param>
|
|||
/// <param name="z">The third value.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract T Invoke(T x, T y, T z); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <param name="z">The third vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector128<T> Invoke(Vector128<T> x, Vector128<T> y, Vector128<T> z); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <param name="z">The third vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector256<T> Invoke(Vector256<T> x, Vector256<T> y, Vector256<T> z); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <param name="z">The third vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector512<T> Invoke(Vector512<T> x, Vector512<T> y, Vector512<T> z); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Validates that an input and destination are either disjoint or begin at the same memory location.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="input">The input values.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void ValidateInputOutputSpanNonOverlapping<T>(ReadOnlySpan<T> input, Span<T> destination) |
|||
{ |
|||
// Runtime TensorPrimitives permits exact same-start overlap for in-place operation. A shifted overlap is
|
|||
// rejected because forward SIMD stores could overwrite input elements before a later load consumes them.
|
|||
if (!Unsafe.AreSame(ref MemoryMarshal.GetReference(input), ref MemoryMarshal.GetReference(destination)) |
|||
&& input.Overlaps(destination)) |
|||
{ |
|||
ThrowInputAndDestinationSpanMustNotOverlap(); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Throws when input spans do not have the same length.
|
|||
/// </summary>
|
|||
[DoesNotReturn] |
|||
private static void ThrowSpansMustHaveSameLength() |
|||
=> throw new ArgumentException("Input span arguments must all have the same length."); |
|||
|
|||
/// <summary>
|
|||
/// Throws when the destination cannot hold every result.
|
|||
/// </summary>
|
|||
[DoesNotReturn] |
|||
private static void ThrowDestinationTooShort() |
|||
=> throw new ArgumentException("Destination is too short.", "destination"); |
|||
|
|||
/// <summary>
|
|||
/// Throws when an input and destination overlap without beginning at the same memory location.
|
|||
/// </summary>
|
|||
[DoesNotReturn] |
|||
private static void ThrowInputAndDestinationSpanMustNotOverlap() |
|||
=> throw new ArgumentException( |
|||
"The destination span may only overlap with an input span if the two spans start at the same memory location.", |
|||
"destination"); |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise binary operation between two spans.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The first input values.</param>
|
|||
/// <param name="y">The second input values.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanSpanIntoSpan<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
ReadOnlySpan<T> y, |
|||
Span<T> destination) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
if (x.Length != y.Length) |
|||
{ |
|||
ThrowSpansMustHaveSameLength(); |
|||
} |
|||
|
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
ValidateInputOutputSpanNonOverlapping(y, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T yRef = ref MemoryMarshal.GetReference(y); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
// Each pipeline preloads its final inputs when a tail overlaps so same-start in-place operation remains correct.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, ref yRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, ref yRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeVectorized128<T, TOperator>(ref xRef, ref yRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), Unsafe.Add(ref yRef, i)); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise binary operation between a span and a scalar.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanScalarIntoSpan<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
T y, |
|||
Span<T> destination) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeVectorized128<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Performs element-wise division using the runtime tensor width-selection order.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The division operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="y">The scalar divisor.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanScalarIntoSpanForDivision<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
T y, |
|||
Span<T> destination) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
// Four values fill one 128-bit float vector. Processing exactly one packed prefix before the scalar
|
|||
// remainder avoids the overlapping second vector that regresses the common seven-element normalization.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector128.IsHardwareAccelerated |
|||
&& Vector128<T>.IsSupported |
|||
&& length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
Vector128<T> yVector = Vector128.Create(y); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef), yVector).StoreUnsafe(ref destinationRef); |
|||
|
|||
for (nuint i = vectorCount; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise ternary operation between a span and two scalars.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanScalarScalarIntoSpan<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
T y, |
|||
T z, |
|||
Span<T> destination) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// This dispatch mirrors the runtime pipeline: large inputs use the widest available registers while
|
|||
// short inputs fall through to a width that fits, keeping the operator contract identical at every length.
|
|||
if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512<T>.IsSupported && length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, y, z, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, y, z, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeVectorized128<T, TOperator>(ref xRef, y, z, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y, z); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between two spans with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first element of the first input.</param>
|
|||
/// <param name="yRef">The first element of the second input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized128<T, TOperator>( |
|||
ref T xRef, |
|||
ref T yRef, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
|
|||
// When a tail exists, both final inputs are loaded before any stores. This permits either source to also
|
|||
// be the destination when the tail starts inside the range written by the preceding full vector.
|
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector128.LoadUnsafe(ref xRef, length - vectorCount), |
|||
Vector128.LoadUnsafe(ref yRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), Vector128.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between two spans with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first element of the first input.</param>
|
|||
/// <param name="yRef">The first element of the second input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized256<T, TOperator>( |
|||
ref T xRef, |
|||
ref T yRef, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector256.LoadUnsafe(ref xRef, length - vectorCount), |
|||
Vector256.LoadUnsafe(ref yRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), Vector256.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between two spans with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first element of the first input.</param>
|
|||
/// <param name="yRef">The first element of the second input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized512<T, TOperator>( |
|||
ref T xRef, |
|||
ref T yRef, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector512.LoadUnsafe(ref xRef, length - vectorCount), |
|||
Vector512.LoadUnsafe(ref yRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), Vector512.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between a span and a scalar with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized128<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector128<T> yVector = Vector128.Create(y); |
|||
|
|||
// When a tail exists, preloading its final vector is required for in-place operation because it must
|
|||
// observe the original values before an earlier overlapping store writes them.
|
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector128.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized256<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> yVector = Vector256.Create(y); |
|||
Vector256<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector256.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized512<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> yVector = Vector512.Create(y); |
|||
Vector512<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector512.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a ternary operation with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized128<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
T z, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector128<T> yVector = Vector128.Create(y); |
|||
Vector128<T> zVector = Vector128.Create(z); |
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector128.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector, |
|||
zVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a ternary operation with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized256<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
T z, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> yVector = Vector256.Create(y); |
|||
Vector256<T> zVector = Vector256.Create(z); |
|||
Vector256<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector256.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector, |
|||
zVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a ternary operation with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized512<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
T z, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> yVector = Vector512.Create(y); |
|||
Vector512<T> zVector = Vector512.Create(z); |
|||
Vector512<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector512.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector, |
|||
zVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
} |
|||
@ -1,249 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise maximum of the values in <paramref name="x"/> and <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The values to compare.</param>
|
|||
/// <param name="y">The value to compare with each element.</param>
|
|||
/// <param name="destination">The destination for the maximum values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Max<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : INumber<T> |
|||
=> InvokeSpanScalarIntoSpan<T, MaxOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<float> MaxSingle(Vector128<float> x, Vector128<float> y) |
|||
{ |
|||
// The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a
|
|||
// first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order.
|
|||
Vector128<float> result = Vector128.Max(x, y); |
|||
result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<float> MaxSingle(Vector256<float> x, Vector256<float> y) |
|||
{ |
|||
Vector256<float> result = Vector256.Max(x, y); |
|||
result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<float> MaxSingle(Vector512<float> x, Vector512<float> y) |
|||
{ |
|||
Vector512<float> result = Vector512.Max(x, y); |
|||
result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<double> MaxDouble(Vector128<double> x, Vector128<double> y) |
|||
{ |
|||
Vector128<double> result = Vector128.Max(x, y); |
|||
result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<double> MaxDouble(Vector256<double> x, Vector256<double> y) |
|||
{ |
|||
Vector256<double> result = Vector256.Max(x, y); |
|||
result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<double> MaxDouble(Vector512<double> x, Vector512<double> y) |
|||
{ |
|||
Vector512<double> result = Vector512.Max(x, y); |
|||
result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum corresponding values.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct MaxOperator<T> : IBinaryOperator<T> |
|||
where T : INumber<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum scalar value.
|
|||
/// </summary>
|
|||
/// <param name="x">The first value.</param>
|
|||
/// <param name="y">The second value.</param>
|
|||
/// <returns>The maximum value.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => T.Max(x, y); |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum values from 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector128<float> result = MaxSingle( |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector128<float>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector128<double> result = MaxDouble( |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector128<double>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
return Vector128.Max(x, y); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum values from 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector256<float> result = MaxSingle( |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector256<float>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector256<double> result = MaxDouble( |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector256<double>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
return Vector256.Max(x, y); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum values from 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector512<float> result = MaxSingle( |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector512<float>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector512<double> result = MaxDouble( |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector512<double>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
return Vector512.Max(x, y); |
|||
} |
|||
} |
|||
} |
|||
@ -1,76 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise product of the values in <paramref name="x"/> and <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multiplier.</param>
|
|||
/// <param name="destination">The destination for the products.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Multiply<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : IMultiplyOperators<T, T, T>, IMultiplicativeIdentity<T, T> |
|||
=> InvokeSpanScalarIntoSpan<T, MultiplyOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Multiplies corresponding values.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct MultiplyOperator<T> : IBinaryOperator<T> |
|||
where T : IMultiplyOperators<T, T, T>, IMultiplicativeIdentity<T, T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicand.</param>
|
|||
/// <param name="y">The multiplier.</param>
|
|||
/// <returns>The product.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => x * y; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multipliers.</param>
|
|||
/// <returns>The products.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) => x * y; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multipliers.</param>
|
|||
/// <returns>The products.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) => x * y; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multipliers.</param>
|
|||
/// <returns>The products.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) => x * y; |
|||
} |
|||
} |
|||
@ -1,332 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Defines an element-wise unary operation.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private interface IUnaryOperator<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether the operation supports vector execution.
|
|||
/// </summary>
|
|||
public static abstract bool Vectorizable { get; } |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a scalar value.
|
|||
/// </summary>
|
|||
/// <param name="x">The input value.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract T Invoke(T x); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a 128-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The input vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector128<T> Invoke(Vector128<T> x); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a 256-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The input vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector256<T> Invoke(Vector256<T> x); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a 512-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The input vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector512<T> Invoke(Vector512<T> x); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Computes the element-wise negation of the values in <paramref name="x"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The values to negate.</param>
|
|||
/// <param name="destination">The destination for the negated values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Negate<T>(ReadOnlySpan<T> x, Span<T> destination) |
|||
where T : IUnaryNegationOperators<T, T> |
|||
=> InvokeSpanIntoSpan<T, NegateOperator<T>>(x, destination); |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise unary operation over a span.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanIntoSpan<T, TOperator>(ReadOnlySpan<T> x, Span<T> destination) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeUnaryVectorized512<T, TOperator>(ref xRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeUnaryVectorized256<T, TOperator>(ref xRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeUnaryVectorized128<T, TOperator>(ref xRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i)); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a unary operation with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeUnaryVectorized128<T, TOperator>(ref T xRef, ref T destinationRef, nuint length) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
|
|||
// The final vector overlaps the preceding store when the length is not a vector multiple. Loading it
|
|||
// before any stores preserves same-start in-place operation because it captures the original tail.
|
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a unary operation with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeUnaryVectorized256<T, TOperator>(ref T xRef, ref T destinationRef, nuint length) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> end = default; |
|||
|
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a unary operation with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeUnaryVectorized512<T, TOperator>(ref T xRef, ref T destinationRef, nuint length) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> end = default; |
|||
|
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Implements element-wise negation for scalar and SIMD inputs.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct NegateOperator<T> : IUnaryOperator<T> |
|||
where T : IUnaryNegationOperators<T, T> |
|||
{ |
|||
/// <inheritdoc />
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <inheritdoc />
|
|||
public static T Invoke(T x) => -x; |
|||
|
|||
/// <inheritdoc />
|
|||
public static Vector128<T> Invoke(Vector128<T> x) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
// IEEE-754 negation toggles the sign bit. Expressing that operation explicitly avoids the
|
|||
// subtraction-based ARM64 code generated by .NET 8 for generic vector negation, which loses
|
|||
// the sign when +0F is negated and therefore differs from both scalar and runtime-main behavior.
|
|||
return x ^ Vector128.Create(-0F).As<float, T>(); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
// Double-precision values use the same sign-bit representation, with the sign in bit 63.
|
|||
return x ^ Vector128.Create(-0D).As<double, T>(); |
|||
} |
|||
|
|||
return -x; |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public static Vector256<T> Invoke(Vector256<T> x) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
// Keep the operation bitwise at every width so ARM64 preserves signed zero exactly.
|
|||
return x ^ Vector256.Create(-0F).As<float, T>(); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
return x ^ Vector256.Create(-0D).As<double, T>(); |
|||
} |
|||
|
|||
return -x; |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public static Vector512<T> Invoke(Vector512<T> x) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
// Vector512 can be hardware accelerated directly or decomposed by the runtime; the explicit
|
|||
// bit operation provides identical IEEE-754 behavior in either case.
|
|||
return x ^ Vector512.Create(-0F).As<float, T>(); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
return x ^ Vector512.Create(-0D).As<double, T>(); |
|||
} |
|||
|
|||
return -x; |
|||
} |
|||
} |
|||
} |
|||
@ -1,580 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using SixLabors.ImageSharp.Common.Helpers; |
|||
using SixLabors.ImageSharp.Tests.TestUtilities; |
|||
|
|||
namespace SixLabors.ImageSharp.Tests.Common; |
|||
|
|||
public class TensorPrimitivesTests |
|||
{ |
|||
private static readonly int[] SpanLengthValues = |
|||
[ |
|||
0, |
|||
1, |
|||
3, |
|||
4, |
|||
5, |
|||
7, |
|||
8, |
|||
9, |
|||
15, |
|||
16, |
|||
17, |
|||
31, |
|||
32, |
|||
33, |
|||
63, |
|||
64, |
|||
65, |
|||
127, |
|||
128, |
|||
129, |
|||
2048 |
|||
]; |
|||
|
|||
/// <summary>
|
|||
/// Gets lengths that exercise scalar execution, every SIMD width, overlapping tails, and the unrolled loop.
|
|||
/// </summary>
|
|||
public static TheoryData<int> SpanLengths => new(SpanLengthValues); |
|||
|
|||
/// <summary>
|
|||
/// Verifies every compatibility operation while forcing the supported SIMD feature tiers in isolated processes.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void OperationsMatchScalarFormulasAcrossHardwareIntrinsicFeatures() |
|||
=> FeatureTestRunner.RunWithHwIntrinsicsFeature( |
|||
RunOperationsAcrossHardwareIntrinsicFeatures, |
|||
HwIntrinsics.AllowAll |
|||
| HwIntrinsics.DisableAVX512F |
|||
| HwIntrinsics.DisableAVX |
|||
| HwIntrinsics.DisableArm64Sve |
|||
| HwIntrinsics.DisableHWIntrinsic); |
|||
|
|||
/// <summary>
|
|||
/// Verifies that span-to-span operations require equal input lengths before accessing either input.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void AddSpanSpanRejectsMismatchedInputLengths() |
|||
{ |
|||
ArgumentException exception = Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(new int[4], new int[3], new int[4])); |
|||
|
|||
Assert.Null(exception.ParamName); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that every operation rejects a destination that cannot hold all input elements.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void OperationsRejectShortDestinations() |
|||
{ |
|||
int[] integers = new int[4]; |
|||
float[] singles = new float[4]; |
|||
|
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Add<int>(integers, integers, new int[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Add<int>(integers, 1, new int[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Multiply<float>(singles, 2F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Divide<float>(singles, 2F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Max<float>(singles, 2F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Clamp<float>(singles, 0F, 1F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Negate<float>(singles, new float[3])).ParamName); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that every operation rejects shifted input and destination overlap.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void OperationsRejectShiftedOverlap() |
|||
{ |
|||
int[] integers = new int[5]; |
|||
int[] separateIntegers = new int[4]; |
|||
float[] singles = new float[5]; |
|||
|
|||
// Both inputs need independent validation because either one may alias a shifted destination.
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(integers.AsSpan(0, 4), separateIntegers, integers.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(separateIntegers, integers.AsSpan(0, 4), integers.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(integers.AsSpan(0, 4), 1, integers.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Multiply<float>(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Divide<float>(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Max<float>(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Clamp<float>(singles.AsSpan(0, 4), 0F, 1F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Negate<float>(singles.AsSpan(0, 4), singles.AsSpan(1, 4))).ParamName); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Runs the TensorPrimitives compatibility assertions inside a process configured for one hardware-intrinsic tier.
|
|||
/// </summary>
|
|||
private static void RunOperationsAcrossHardwareIntrinsicFeatures() |
|||
{ |
|||
TensorPrimitivesTests tests = new(); |
|||
|
|||
// Reuse the focused assertions so the remote feature matrix cannot drift from the normal test coverage.
|
|||
foreach (int length in SpanLengthValues) |
|||
{ |
|||
tests.AddByteMatchesScalarFormula(length); |
|||
tests.AddUInt32MatchesScalarFormula(length); |
|||
tests.AddScalarInt32MatchesScalarFormula(length); |
|||
tests.NegateSingleMatchesScalarFormula(length); |
|||
tests.NegateDoubleMatchesScalarFormula(length); |
|||
tests.ClampInt32MatchesScalarFormula(length); |
|||
tests.ClampSingleMatchesRuntimeFormula(length); |
|||
tests.DivideSingleMatchesScalarFormula(length); |
|||
tests.MaxSingleMatchesRuntimeFormula(length); |
|||
tests.MultiplySingleMatchesScalarFormula(length); |
|||
tests.NormalizeMatchesScalarFormula(length); |
|||
} |
|||
|
|||
tests.ClampSinglePreservesRuntimeSpecialValueSemantics(); |
|||
tests.ClampDoublePreservesRuntimeSpecialValueSemantics(); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that byte addition wraps modulo 256 and supports either input as the in-place destination.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void AddByteMatchesScalarFormula(int length) |
|||
{ |
|||
byte[] x = new byte[length]; |
|||
byte[] y = new byte[length]; |
|||
byte[] expected = new byte[length]; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
x[i] = (byte)((i * 23) + 197); |
|||
y[i] = (byte)((i * 41) + 113); |
|||
expected[i] = unchecked((byte)(x[i] + y[i])); |
|||
} |
|||
|
|||
byte[] destination = new byte[length]; |
|||
TensorPrimitives_.Add<byte>(x, y, destination); |
|||
Assert.Equal(expected, destination); |
|||
|
|||
byte[] xInPlace = (byte[])x.Clone(); |
|||
TensorPrimitives_.Add<byte>(xInPlace, y, xInPlace); |
|||
Assert.Equal(expected, xInPlace); |
|||
|
|||
byte[] yInPlace = (byte[])y.Clone(); |
|||
TensorPrimitives_.Add<byte>(x, yInPlace, yInPlace); |
|||
Assert.Equal(expected, yInPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that unsigned integer addition preserves unchecked histogram accumulation semantics.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void AddUInt32MatchesScalarFormula(int length) |
|||
{ |
|||
uint[] x = new uint[length]; |
|||
uint[] y = new uint[length]; |
|||
uint[] expected = new uint[length]; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
x[i] = ((uint)i * 1_234_567U) + 0xF0000000U; |
|||
y[i] = ((uint)i * 7_654_321U) + 0x30000000U; |
|||
expected[i] = unchecked(x[i] + y[i]); |
|||
} |
|||
|
|||
TensorPrimitives_.Add<uint>(x, y, x); |
|||
Assert.Equal(expected, x); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that scalar integer addition produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void AddScalarInt32MatchesScalarFormula(int length) |
|||
{ |
|||
int[] source = new int[length]; |
|||
int[] expected = new int[length]; |
|||
const int addend = 17; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
source[i] = (i * 37) - 200; |
|||
expected[i] = source[i] + addend; |
|||
} |
|||
|
|||
int[] destination = new int[length]; |
|||
TensorPrimitives_.Add(source, addend, destination); |
|||
Assert.Equal(expected, destination); |
|||
|
|||
int[] inPlace = (int[])source.Clone(); |
|||
TensorPrimitives_.Add(inPlace, addend, inPlace); |
|||
Assert.Equal(expected, inPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that floating-point negation preserves the scalar operator's exact bit-level behavior.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void NegateSingleMatchesScalarFormula(int length) |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
-0F, |
|||
0F, |
|||
-1F, |
|||
1F, |
|||
float.NegativeInfinity, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = values[i % values.Length]; |
|||
expected[i] = -source[i]; |
|||
} |
|||
|
|||
float[] destination = new float[length]; |
|||
TensorPrimitives_.Negate<float>(source, destination); |
|||
AssertSingleBitsEqual(expected, destination); |
|||
|
|||
TensorPrimitives_.Negate<float>(source, source); |
|||
AssertSingleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that double-precision negation preserves the scalar operator's exact bit-level behavior.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void NegateDoubleMatchesScalarFormula(int length) |
|||
{ |
|||
double[] values = |
|||
[ |
|||
double.NaN, |
|||
-0D, |
|||
0D, |
|||
-1D, |
|||
1D, |
|||
double.NegativeInfinity, |
|||
double.PositiveInfinity |
|||
]; |
|||
|
|||
double[] source = new double[length]; |
|||
double[] expected = new double[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = values[i % values.Length]; |
|||
expected[i] = -source[i]; |
|||
} |
|||
|
|||
double[] destination = new double[length]; |
|||
TensorPrimitives_.Negate<double>(source, destination); |
|||
AssertDoubleBitsEqual(expected, destination); |
|||
|
|||
TensorPrimitives_.Negate<double>(source, source); |
|||
AssertDoubleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that integer clamping produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void ClampInt32MatchesScalarFormula(int length) |
|||
{ |
|||
int[] source = new int[length]; |
|||
int[] expected = new int[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = ((i * 37) % 401) - 200; |
|||
expected[i] = Math.Clamp(source[i], -73, 91); |
|||
} |
|||
|
|||
int[] destination = new int[length]; |
|||
TensorPrimitives_.Clamp<int>(source, -73, 91, destination); |
|||
Assert.Equal(expected, destination); |
|||
|
|||
int[] inPlace = (int[])source.Clone(); |
|||
TensorPrimitives_.Clamp<int>(inPlace, -73, 91, inPlace); |
|||
Assert.Equal(expected, inPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that floating-point clamping matches the runtime tensor formula for special values and unordered bounds.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void ClampSingleMatchesRuntimeFormula(int length) |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
-0F, |
|||
0F, |
|||
-1F, |
|||
1F, |
|||
float.NegativeInfinity, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = values[i % values.Length]; |
|||
|
|||
// Runtime main follows Min(Max(x, min), max) for vectorizable types, including unordered bounds.
|
|||
expected[i] = float.Min(float.Max(source[i], 2F), -2F); |
|||
} |
|||
|
|||
TensorPrimitives_.Clamp<float>(source, 2F, -2F, source); |
|||
AssertSingleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that single-precision clamping preserves the runtime's signed-zero and NaN behavior.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void ClampSinglePreservesRuntimeSpecialValueSemantics() |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
float.NegativeInfinity, |
|||
-0F, |
|||
0F, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] actual = new float[129]; |
|||
float[] expected = new float[actual.Length]; |
|||
|
|||
for (int i = 0; i < actual.Length; i++) |
|||
{ |
|||
actual[i] = values[i % values.Length]; |
|||
expected[i] = float.Min(float.Max(actual[i], -0F), 0F); |
|||
} |
|||
|
|||
TensorPrimitives_.Clamp<float>(actual, -0F, 0F, actual); |
|||
AssertSingleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that double-precision clamping preserves the runtime's signed-zero and NaN behavior.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void ClampDoublePreservesRuntimeSpecialValueSemantics() |
|||
{ |
|||
double[] values = |
|||
[ |
|||
double.NaN, |
|||
double.NegativeInfinity, |
|||
-0D, |
|||
0D, |
|||
double.PositiveInfinity |
|||
]; |
|||
|
|||
double[] actual = new double[65]; |
|||
double[] expected = new double[actual.Length]; |
|||
|
|||
for (int i = 0; i < actual.Length; i++) |
|||
{ |
|||
actual[i] = values[i % values.Length]; |
|||
expected[i] = double.Min(double.Max(actual[i], -0D), 0D); |
|||
} |
|||
|
|||
TensorPrimitives_.Clamp<double>(actual, -0D, 0D, actual); |
|||
AssertDoubleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that division produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void DivideSingleMatchesScalarFormula(int length) |
|||
{ |
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = (i - 65.25F) * 1.75F; |
|||
expected[i] = source[i] / 3.25F; |
|||
} |
|||
|
|||
float[] destination = new float[length]; |
|||
TensorPrimitives_.Divide<float>(source, 3.25F, destination); |
|||
AssertSingleBitsEqual(expected, destination); |
|||
|
|||
float[] inPlace = (float[])source.Clone(); |
|||
TensorPrimitives_.Divide<float>(inPlace, 3.25F, inPlace); |
|||
AssertSingleBitsEqual(expected, inPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that maximum selection preserves the runtime's NaN and signed-zero semantics.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void MaxSingleMatchesRuntimeFormula(int length) |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
float.NegativeInfinity, |
|||
-1F, |
|||
-0F, |
|||
0F, |
|||
1F, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] actual = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
actual[i] = values[i % values.Length]; |
|||
expected[i] = float.Max(actual[i], -0F); |
|||
} |
|||
|
|||
TensorPrimitives_.Max<float>(actual, -0F, actual); |
|||
AssertSingleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that multiplication produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void MultiplySingleMatchesScalarFormula(int length) |
|||
{ |
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = (i - 65.25F) * 1.75F; |
|||
expected[i] = source[i] * 0.375F; |
|||
} |
|||
|
|||
float[] destination = new float[length]; |
|||
TensorPrimitives_.Multiply<float>(source, 0.375F, destination); |
|||
AssertSingleBitsEqual(expected, destination); |
|||
|
|||
TensorPrimitives_.Multiply<float>(source, 0.375F, source); |
|||
AssertSingleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that the normalization compatibility call preserves its element-wise division contract.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void NormalizeMatchesScalarFormula(int length) |
|||
{ |
|||
float[] actual = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < actual.Length; i++) |
|||
{ |
|||
actual[i] = (i + 1) * 0.125F; |
|||
expected[i] = actual[i] / 7.5F; |
|||
} |
|||
|
|||
Numerics.Normalize(actual, 7.5F); |
|||
AssertSingleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Compares floating-point results while preserving signed-zero behavior.
|
|||
/// </summary>
|
|||
/// <param name="expected">The expected values.</param>
|
|||
/// <param name="actual">The actual values.</param>
|
|||
private static void AssertSingleBitsEqual(ReadOnlySpan<float> expected, ReadOnlySpan<float> actual) |
|||
{ |
|||
Assert.Equal(expected.Length, actual.Length); |
|||
|
|||
for (int i = 0; i < expected.Length; i++) |
|||
{ |
|||
if (float.IsNaN(expected[i])) |
|||
{ |
|||
Assert.True(float.IsNaN(actual[i])); |
|||
} |
|||
else |
|||
{ |
|||
Assert.Equal(BitConverter.SingleToInt32Bits(expected[i]), BitConverter.SingleToInt32Bits(actual[i])); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Compares double-precision results while preserving signed-zero behavior.
|
|||
/// </summary>
|
|||
/// <param name="expected">The expected values.</param>
|
|||
/// <param name="actual">The actual values.</param>
|
|||
private static void AssertDoubleBitsEqual(ReadOnlySpan<double> expected, ReadOnlySpan<double> actual) |
|||
{ |
|||
Assert.Equal(expected.Length, actual.Length); |
|||
|
|||
for (int i = 0; i < expected.Length; i++) |
|||
{ |
|||
if (double.IsNaN(expected[i])) |
|||
{ |
|||
Assert.True(double.IsNaN(actual[i])); |
|||
} |
|||
else |
|||
{ |
|||
Assert.Equal(BitConverter.DoubleToInt64Bits(expected[i]), BitConverter.DoubleToInt64Bits(actual[i])); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue