mirror of https://github.com/SixLabors/ImageSharp
James Jackson-South
1 week ago
committed by
GitHub
GitHub
52 changed files with 343 additions and 3756 deletions
@ -1,93 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise sum of the values in <paramref name="x"/> and <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <param name="destination">The destination for the sums.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="x"/> and <paramref name="y"/> do not have the same length.</exception>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than the input spans.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// An input and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Add<T>(ReadOnlySpan<T> x, ReadOnlySpan<T> y, Span<T> destination) |
|||
where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T> |
|||
=> InvokeSpanSpanIntoSpan<T, AddOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Computes the element-wise sum of the values in <paramref name="x"/> and the scalar <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The scalar second addend.</param>
|
|||
/// <param name="destination">The destination for the sums.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Add<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T> |
|||
=> InvokeSpanScalarIntoSpan<T, AddOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Adds corresponding values.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct AddOperator<T> : IBinaryOperator<T> |
|||
where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Adds scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addend.</param>
|
|||
/// <param name="y">The second addend.</param>
|
|||
/// <returns>The sum.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => x + y; |
|||
|
|||
/// <summary>
|
|||
/// Adds 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <returns>The sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) => x + y; |
|||
|
|||
/// <summary>
|
|||
/// Adds 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <returns>The sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) => x + y; |
|||
|
|||
/// <summary>
|
|||
/// Adds 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first addends.</param>
|
|||
/// <param name="y">The second addends.</param>
|
|||
/// <returns>The sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) => x + y; |
|||
} |
|||
} |
|||
@ -1,322 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise result of clamping <paramref name="x"/> to the inclusive range specified
|
|||
/// by <paramref name="min"/> and <paramref name="max"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bound.</param>
|
|||
/// <param name="max">The inclusive upper bound.</param>
|
|||
/// <param name="destination">The destination for the clamped values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Clamp<T>(ReadOnlySpan<T> x, T min, T max, Span<T> destination) |
|||
where T : INumber<T> |
|||
=> InvokeSpanScalarScalarIntoSpan<T, ClampOperator<T>>(x, min, max, destination); |
|||
|
|||
/// <summary>
|
|||
/// Clamps single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<float> ClampSingle( |
|||
Vector128<float> value, |
|||
Vector128<float> min, |
|||
Vector128<float> max) |
|||
{ |
|||
// Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the
|
|||
// first operand and select negative zero when equal values have different signs.
|
|||
Vector128<float> maximum = Vector128.ConditionalSelect( |
|||
Vector128.LessThan(min, value) |
|||
| ~Vector128.Equals(value, value) |
|||
| (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.LessThan(maximum, max) |
|||
| ~Vector128.Equals(maximum, maximum) |
|||
| (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<float> ClampSingle( |
|||
Vector256<float> value, |
|||
Vector256<float> min, |
|||
Vector256<float> max) |
|||
{ |
|||
Vector256<float> maximum = Vector256.ConditionalSelect( |
|||
Vector256.LessThan(min, value) |
|||
| ~Vector256.Equals(value, value) |
|||
| (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.LessThan(maximum, max) |
|||
| ~Vector256.Equals(maximum, maximum) |
|||
| (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<float> ClampSingle( |
|||
Vector512<float> value, |
|||
Vector512<float> min, |
|||
Vector512<float> max) |
|||
{ |
|||
Vector512<float> maximum = Vector512.ConditionalSelect( |
|||
Vector512.LessThan(min, value) |
|||
| ~Vector512.Equals(value, value) |
|||
| (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.LessThan(maximum, max) |
|||
| ~Vector512.Equals(maximum, maximum) |
|||
| (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<double> ClampDouble( |
|||
Vector128<double> value, |
|||
Vector128<double> min, |
|||
Vector128<double> max) |
|||
{ |
|||
Vector128<double> maximum = Vector128.ConditionalSelect( |
|||
Vector128.LessThan(min, value) |
|||
| ~Vector128.Equals(value, value) |
|||
| (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.LessThan(maximum, max) |
|||
| ~Vector128.Equals(maximum, maximum) |
|||
| (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<double> ClampDouble( |
|||
Vector256<double> value, |
|||
Vector256<double> min, |
|||
Vector256<double> max) |
|||
{ |
|||
Vector256<double> maximum = Vector256.ConditionalSelect( |
|||
Vector256.LessThan(min, value) |
|||
| ~Vector256.Equals(value, value) |
|||
| (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.LessThan(maximum, max) |
|||
| ~Vector256.Equals(maximum, maximum) |
|||
| (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="value">The values to clamp.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<double> ClampDouble( |
|||
Vector512<double> value, |
|||
Vector512<double> min, |
|||
Vector512<double> max) |
|||
{ |
|||
Vector512<double> maximum = Vector512.ConditionalSelect( |
|||
Vector512.LessThan(min, value) |
|||
| ~Vector512.Equals(value, value) |
|||
| (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), |
|||
value, |
|||
min); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.LessThan(maximum, max) |
|||
| ~Vector512.Equals(maximum, maximum) |
|||
| (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), |
|||
maximum, |
|||
max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps values using the complete runtime tensor contract, including signed-zero correction.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct ClampOperator<T> : ITernaryOperator<T> |
|||
where T : INumber<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Clamps a scalar value.
|
|||
/// </summary>
|
|||
/// <param name="x">The value.</param>
|
|||
/// <param name="min">The inclusive lower bound.</param>
|
|||
/// <param name="max">The inclusive upper bound.</param>
|
|||
/// <returns>The clamped value.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T min, T max) |
|||
=> Vector128<T>.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max); |
|||
|
|||
/// <summary>
|
|||
/// Clamps a 128-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The values.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> min, Vector128<T> max) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector128<float> result = ClampSingle( |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref min), |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector128<float>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector128<double> result = ClampDouble( |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref min), |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector128<double>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
return Vector128_.Clamp(x, min, max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps a 256-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The values.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> min, Vector256<T> max) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector256<float> result = ClampSingle( |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref min), |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector256<float>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector256<double> result = ClampDouble( |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref min), |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector256<double>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
return Vector256_.Clamp(x, min, max); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Clamps a 512-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The values.</param>
|
|||
/// <param name="min">The inclusive lower bounds.</param>
|
|||
/// <param name="max">The inclusive upper bounds.</param>
|
|||
/// <returns>The clamped values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> min, Vector512<T> max) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector512<float> result = ClampSingle( |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref min), |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector512<float>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector512<double> result = ClampDouble( |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref min), |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref max)); |
|||
|
|||
return Unsafe.As<Vector512<double>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
return Vector512_.Clamp(x, min, max); |
|||
} |
|||
} |
|||
} |
|||
@ -1,87 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise result of dividing the values in <paramref name="x"/> by <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The dividend values.</param>
|
|||
/// <param name="y">The divisor.</param>
|
|||
/// <param name="destination">The destination for the quotient values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Divide<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : IDivisionOperators<T, T, T> |
|||
=> InvokeSpanScalarIntoSpanForDivision<T, DivideOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Determines whether <typeparamref name="T"/> has the same vector division support as <see cref="int"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <returns><see langword="true"/> when <typeparamref name="T"/> is a 32-bit signed native integer type.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static bool IsInt32Like<T>() |
|||
=> typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); |
|||
|
|||
/// <summary>
|
|||
/// Divides values by a scalar.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct DivideOperator<T> : IBinaryOperator<T> |
|||
where T : IDivisionOperators<T, T, T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => typeof(T) == typeof(float) |
|||
|| typeof(T) == typeof(double) |
|||
|| (Vector256.IsHardwareAccelerated && IsInt32Like<T>()); |
|||
|
|||
/// <summary>
|
|||
/// Divides scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividend.</param>
|
|||
/// <param name="y">The divisor.</param>
|
|||
/// <returns>The quotient.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => x / y; |
|||
|
|||
/// <summary>
|
|||
/// Divides 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividends.</param>
|
|||
/// <param name="y">The divisors.</param>
|
|||
/// <returns>The quotients.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) => x / y; |
|||
|
|||
/// <summary>
|
|||
/// Divides 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividends.</param>
|
|||
/// <param name="y">The divisors.</param>
|
|||
/// <returns>The quotients.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) => x / y; |
|||
|
|||
/// <summary>
|
|||
/// Divides 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The dividends.</param>
|
|||
/// <param name="y">The divisors.</param>
|
|||
/// <returns>The quotients.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) => x / y; |
|||
} |
|||
} |
|||
@ -1,900 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Diagnostics.CodeAnalysis; |
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
/// <summary>
|
|||
/// Provides compatibility implementations for tensor operations that are not available on every target framework.
|
|||
/// </summary>
|
|||
/// <remarks>
|
|||
/// The API shape follows <c>System.Numerics.Tensors.TensorPrimitives</c> so call sites can move to the runtime
|
|||
/// implementation when ImageSharp no longer supports target frameworks that predate it.
|
|||
/// </remarks>
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Defines an element-wise binary operation.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private interface IBinaryOperator<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether the operation supports vector execution.
|
|||
/// </summary>
|
|||
public static abstract bool Vectorizable { get; } |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The first value.</param>
|
|||
/// <param name="y">The second value.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract T Invoke(T x, T y); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector128<T> Invoke(Vector128<T> x, Vector128<T> y); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector256<T> Invoke(Vector256<T> x, Vector256<T> y); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector512<T> Invoke(Vector512<T> x, Vector512<T> y); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Defines an element-wise ternary operation.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private interface ITernaryOperator<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether the operation supports vector execution.
|
|||
/// </summary>
|
|||
public static abstract bool Vectorizable { get; } |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The first value.</param>
|
|||
/// <param name="y">The second value.</param>
|
|||
/// <param name="z">The third value.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract T Invoke(T x, T y, T z); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <param name="z">The third vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector128<T> Invoke(Vector128<T> x, Vector128<T> y, Vector128<T> z); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <param name="z">The third vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector256<T> Invoke(Vector256<T> x, Vector256<T> y, Vector256<T> z); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first vector.</param>
|
|||
/// <param name="y">The second vector.</param>
|
|||
/// <param name="z">The third vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector512<T> Invoke(Vector512<T> x, Vector512<T> y, Vector512<T> z); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Validates that an input and destination are either disjoint or begin at the same memory location.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="input">The input values.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void ValidateInputOutputSpanNonOverlapping<T>(ReadOnlySpan<T> input, Span<T> destination) |
|||
{ |
|||
// Runtime TensorPrimitives permits exact same-start overlap for in-place operation. A shifted overlap is
|
|||
// rejected because forward SIMD stores could overwrite input elements before a later load consumes them.
|
|||
if (!Unsafe.AreSame(ref MemoryMarshal.GetReference(input), ref MemoryMarshal.GetReference(destination)) |
|||
&& input.Overlaps(destination)) |
|||
{ |
|||
ThrowInputAndDestinationSpanMustNotOverlap(); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Throws when input spans do not have the same length.
|
|||
/// </summary>
|
|||
[DoesNotReturn] |
|||
private static void ThrowSpansMustHaveSameLength() |
|||
=> throw new ArgumentException("Input span arguments must all have the same length."); |
|||
|
|||
/// <summary>
|
|||
/// Throws when the destination cannot hold every result.
|
|||
/// </summary>
|
|||
[DoesNotReturn] |
|||
private static void ThrowDestinationTooShort() |
|||
=> throw new ArgumentException("Destination is too short.", "destination"); |
|||
|
|||
/// <summary>
|
|||
/// Throws when an input and destination overlap without beginning at the same memory location.
|
|||
/// </summary>
|
|||
[DoesNotReturn] |
|||
private static void ThrowInputAndDestinationSpanMustNotOverlap() |
|||
=> throw new ArgumentException( |
|||
"The destination span may only overlap with an input span if the two spans start at the same memory location.", |
|||
"destination"); |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise binary operation between two spans.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The first input values.</param>
|
|||
/// <param name="y">The second input values.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanSpanIntoSpan<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
ReadOnlySpan<T> y, |
|||
Span<T> destination) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
if (x.Length != y.Length) |
|||
{ |
|||
ThrowSpansMustHaveSameLength(); |
|||
} |
|||
|
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
ValidateInputOutputSpanNonOverlapping(y, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T yRef = ref MemoryMarshal.GetReference(y); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
// Each pipeline preloads its final inputs when a tail overlaps so same-start in-place operation remains correct.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, ref yRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, ref yRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeVectorized128<T, TOperator>(ref xRef, ref yRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), Unsafe.Add(ref yRef, i)); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise binary operation between a span and a scalar.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanScalarIntoSpan<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
T y, |
|||
Span<T> destination) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeVectorized128<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Performs element-wise division using the runtime tensor width-selection order.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The division operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="y">The scalar divisor.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanScalarIntoSpanForDivision<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
T y, |
|||
Span<T> destination) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, y, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
// Four values fill one 128-bit float vector. Processing exactly one packed prefix before the scalar
|
|||
// remainder avoids the overlapping second vector that regresses the common seven-element normalization.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector128.IsHardwareAccelerated |
|||
&& Vector128<T>.IsSupported |
|||
&& length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
Vector128<T> yVector = Vector128.Create(y); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef), yVector).StoreUnsafe(ref destinationRef); |
|||
|
|||
for (nuint i = vectorCount; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise ternary operation between a span and two scalars.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanScalarScalarIntoSpan<T, TOperator>( |
|||
ReadOnlySpan<T> x, |
|||
T y, |
|||
T z, |
|||
Span<T> destination) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// This dispatch mirrors the runtime pipeline: large inputs use the widest available registers while
|
|||
// short inputs fall through to a width that fits, keeping the operator contract identical at every length.
|
|||
if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512<T>.IsSupported && length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeVectorized512<T, TOperator>(ref xRef, y, z, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeVectorized256<T, TOperator>(ref xRef, y, z, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeVectorized128<T, TOperator>(ref xRef, y, z, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y, z); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between two spans with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first element of the first input.</param>
|
|||
/// <param name="yRef">The first element of the second input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized128<T, TOperator>( |
|||
ref T xRef, |
|||
ref T yRef, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
|
|||
// When a tail exists, both final inputs are loaded before any stores. This permits either source to also
|
|||
// be the destination when the tail starts inside the range written by the preceding full vector.
|
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector128.LoadUnsafe(ref xRef, length - vectorCount), |
|||
Vector128.LoadUnsafe(ref yRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), Vector128.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between two spans with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first element of the first input.</param>
|
|||
/// <param name="yRef">The first element of the second input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized256<T, TOperator>( |
|||
ref T xRef, |
|||
ref T yRef, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector256.LoadUnsafe(ref xRef, length - vectorCount), |
|||
Vector256.LoadUnsafe(ref yRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), Vector256.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between two spans with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first element of the first input.</param>
|
|||
/// <param name="yRef">The first element of the second input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized512<T, TOperator>( |
|||
ref T xRef, |
|||
ref T yRef, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector512.LoadUnsafe(ref xRef, length - vectorCount), |
|||
Vector512.LoadUnsafe(ref yRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), Vector512.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation between a span and a scalar with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized128<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector128<T> yVector = Vector128.Create(y); |
|||
|
|||
// When a tail exists, preloading its final vector is required for in-place operation because it must
|
|||
// observe the original values before an earlier overlapping store writes them.
|
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector128.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized256<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> yVector = Vector256.Create(y); |
|||
Vector256<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector256.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a binary operation with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized512<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, IBinaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> yVector = Vector512.Create(y); |
|||
Vector512<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector512.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a ternary operation with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized128<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
T z, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector128<T> yVector = Vector128.Create(y); |
|||
Vector128<T> zVector = Vector128.Create(z); |
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector128.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector, |
|||
zVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a ternary operation with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized256<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
T z, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> yVector = Vector256.Create(y); |
|||
Vector256<T> zVector = Vector256.Create(z); |
|||
Vector256<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector256.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector, |
|||
zVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a ternary operation with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="y">The first scalar input.</param>
|
|||
/// <param name="z">The second scalar input.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeVectorized512<T, TOperator>( |
|||
ref T xRef, |
|||
T y, |
|||
T z, |
|||
ref T destinationRef, |
|||
nuint length) |
|||
where TOperator : struct, ITernaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> yVector = Vector512.Create(y); |
|||
Vector512<T> zVector = Vector512.Create(z); |
|||
Vector512<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke( |
|||
Vector512.LoadUnsafe(ref xRef, length - vectorCount), |
|||
yVector, |
|||
zVector); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
} |
|||
@ -1,249 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise maximum of the values in <paramref name="x"/> and <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The values to compare.</param>
|
|||
/// <param name="y">The value to compare with each element.</param>
|
|||
/// <param name="destination">The destination for the maximum values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Max<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : INumber<T> |
|||
=> InvokeSpanScalarIntoSpan<T, MaxOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<float> MaxSingle(Vector128<float> x, Vector128<float> y) |
|||
{ |
|||
// The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a
|
|||
// first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order.
|
|||
Vector128<float> result = Vector128.Max(x, y); |
|||
result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<float> MaxSingle(Vector256<float> x, Vector256<float> y) |
|||
{ |
|||
Vector256<float> result = Vector256.Max(x, y); |
|||
result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum single-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<float> MaxSingle(Vector512<float> x, Vector512<float> y) |
|||
{ |
|||
Vector512<float> result = Vector512.Max(x, y); |
|||
result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<double> MaxDouble(Vector128<double> x, Vector128<double> y) |
|||
{ |
|||
Vector128<double> result = Vector128.Max(x, y); |
|||
result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); |
|||
|
|||
return Vector128.ConditionalSelect( |
|||
Vector128.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<double> MaxDouble(Vector256<double> x, Vector256<double> y) |
|||
{ |
|||
Vector256<double> result = Vector256.Max(x, y); |
|||
result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); |
|||
|
|||
return Vector256.ConditionalSelect( |
|||
Vector256.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects maximum double-precision values with the normalized runtime semantics.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector512<double> MaxDouble(Vector512<double> x, Vector512<double> y) |
|||
{ |
|||
Vector512<double> result = Vector512.Max(x, y); |
|||
result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); |
|||
|
|||
return Vector512.ConditionalSelect( |
|||
Vector512.Equals(x, y), |
|||
x & y, |
|||
result); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum corresponding values.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct MaxOperator<T> : IBinaryOperator<T> |
|||
where T : INumber<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum scalar value.
|
|||
/// </summary>
|
|||
/// <param name="x">The first value.</param>
|
|||
/// <param name="y">The second value.</param>
|
|||
/// <returns>The maximum value.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => T.Max(x, y); |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum values from 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector128<float> result = MaxSingle( |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<float>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector128<float>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector128<double> result = MaxDouble( |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref x), |
|||
Unsafe.As<Vector128<T>, Vector128<double>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector128<double>, Vector128<T>>(ref result); |
|||
} |
|||
|
|||
return Vector128.Max(x, y); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum values from 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector256<float> result = MaxSingle( |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<float>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector256<float>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector256<double> result = MaxDouble( |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref x), |
|||
Unsafe.As<Vector256<T>, Vector256<double>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector256<double>, Vector256<T>>(ref result); |
|||
} |
|||
|
|||
return Vector256.Max(x, y); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Selects the maximum values from 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The first values.</param>
|
|||
/// <param name="y">The second values.</param>
|
|||
/// <returns>The maximum values.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
Vector512<float> result = MaxSingle( |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<float>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector512<float>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
Vector512<double> result = MaxDouble( |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref x), |
|||
Unsafe.As<Vector512<T>, Vector512<double>>(ref y)); |
|||
|
|||
return Unsafe.As<Vector512<double>, Vector512<T>>(ref result); |
|||
} |
|||
|
|||
return Vector512.Max(x, y); |
|||
} |
|||
} |
|||
} |
|||
@ -1,76 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Computes the element-wise product of the values in <paramref name="x"/> and <paramref name="y"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multiplier.</param>
|
|||
/// <param name="destination">The destination for the products.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Multiply<T>(ReadOnlySpan<T> x, T y, Span<T> destination) |
|||
where T : IMultiplyOperators<T, T, T>, IMultiplicativeIdentity<T, T> |
|||
=> InvokeSpanScalarIntoSpan<T, MultiplyOperator<T>>(x, y, destination); |
|||
|
|||
/// <summary>
|
|||
/// Multiplies corresponding values.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct MultiplyOperator<T> : IBinaryOperator<T> |
|||
where T : IMultiplyOperators<T, T, T>, IMultiplicativeIdentity<T, T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether this operation supports vector execution.
|
|||
/// </summary>
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies scalar values.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicand.</param>
|
|||
/// <param name="y">The multiplier.</param>
|
|||
/// <returns>The product.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static T Invoke(T x, T y) => x * y; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multipliers.</param>
|
|||
/// <returns>The products.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> y) => x * y; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multipliers.</param>
|
|||
/// <returns>The products.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> y) => x * y; |
|||
|
|||
/// <summary>
|
|||
/// Multiplies 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <param name="x">The multiplicands.</param>
|
|||
/// <param name="y">The multipliers.</param>
|
|||
/// <returns>The products.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<T> Invoke(Vector512<T> x, Vector512<T> y) => x * y; |
|||
} |
|||
} |
|||
@ -1,332 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Runtime.Intrinsics; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
internal static partial class TensorPrimitives_ |
|||
{ |
|||
/// <summary>
|
|||
/// Defines an element-wise unary operation.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private interface IUnaryOperator<T> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether the operation supports vector execution.
|
|||
/// </summary>
|
|||
public static abstract bool Vectorizable { get; } |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a scalar value.
|
|||
/// </summary>
|
|||
/// <param name="x">The input value.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract T Invoke(T x); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a 128-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The input vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector128<T> Invoke(Vector128<T> x); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a 256-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The input vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector256<T> Invoke(Vector256<T> x); |
|||
|
|||
/// <summary>
|
|||
/// Applies the operation to a 512-bit vector.
|
|||
/// </summary>
|
|||
/// <param name="x">The input vector.</param>
|
|||
/// <returns>The operation result.</returns>
|
|||
public static abstract Vector512<T> Invoke(Vector512<T> x); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Computes the element-wise negation of the values in <paramref name="x"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <param name="x">The values to negate.</param>
|
|||
/// <param name="destination">The destination for the negated values.</param>
|
|||
/// <exception cref="ArgumentException"><paramref name="destination"/> is shorter than <paramref name="x"/>.</exception>
|
|||
/// <exception cref="ArgumentException">
|
|||
/// <paramref name="x"/> and <paramref name="destination"/> overlap without beginning at the same memory location.
|
|||
/// </exception>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static void Negate<T>(ReadOnlySpan<T> x, Span<T> destination) |
|||
where T : IUnaryNegationOperators<T, T> |
|||
=> InvokeSpanIntoSpan<T, NegateOperator<T>>(x, destination); |
|||
|
|||
/// <summary>
|
|||
/// Performs an element-wise unary operation over a span.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="x">The input values.</param>
|
|||
/// <param name="destination">The destination values.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeSpanIntoSpan<T, TOperator>(ReadOnlySpan<T> x, Span<T> destination) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
if (x.Length > destination.Length) |
|||
{ |
|||
ThrowDestinationTooShort(); |
|||
} |
|||
|
|||
ValidateInputOutputSpanNonOverlapping(x, destination); |
|||
|
|||
ref T xRef = ref MemoryMarshal.GetReference(x); |
|||
ref T destinationRef = ref MemoryMarshal.GetReference(destination); |
|||
nuint length = (uint)x.Length; |
|||
|
|||
// Runtime main selects the widest supported pipeline once one complete vector is available.
|
|||
if (TOperator.Vectorizable |
|||
&& Vector512.IsHardwareAccelerated |
|||
&& Vector512<T>.IsSupported |
|||
&& length >= (uint)Vector512<T>.Count) |
|||
{ |
|||
InvokeUnaryVectorized512<T, TOperator>(ref xRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && length >= (uint)Vector256<T>.Count) |
|||
{ |
|||
InvokeUnaryVectorized256<T, TOperator>(ref xRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && length >= (uint)Vector128<T>.Count) |
|||
{ |
|||
InvokeUnaryVectorized128<T, TOperator>(ref xRef, ref destinationRef, length); |
|||
return; |
|||
} |
|||
|
|||
for (nuint i = 0; i < length; i++) |
|||
{ |
|||
Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i)); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a unary operation with 128-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeUnaryVectorized128<T, TOperator>(ref T xRef, ref T destinationRef, nuint length) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector128<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
|
|||
// The final vector overlaps the preceding store when the length is not a vector multiple. Loading it
|
|||
// before any stores preserves same-start in-place operation because it captures the original tail.
|
|||
Vector128<T> end = default; |
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a unary operation with 256-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeUnaryVectorized256<T, TOperator>(ref T xRef, ref T destinationRef, nuint length) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector256<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector256<T> end = default; |
|||
|
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies a unary operation with 512-bit vectors.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
/// <typeparam name="TOperator">The operation to apply.</typeparam>
|
|||
/// <param name="xRef">The first input element.</param>
|
|||
/// <param name="destinationRef">The first destination element.</param>
|
|||
/// <param name="length">The number of elements to process.</param>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static void InvokeUnaryVectorized512<T, TOperator>(ref T xRef, ref T destinationRef, nuint length) |
|||
where TOperator : struct, IUnaryOperator<T> |
|||
{ |
|||
nuint vectorCount = (uint)Vector512<T>.Count; |
|||
nuint vectorsPerLoop = vectorCount * 8; |
|||
nuint index = 0; |
|||
Vector512<T> end = default; |
|||
|
|||
if ((length % vectorCount) != 0) |
|||
{ |
|||
end = TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, length - vectorCount)); |
|||
} |
|||
|
|||
while ((length - index) >= vectorsPerLoop) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); |
|||
|
|||
index += vectorsPerLoop; |
|||
} |
|||
|
|||
while ((length - index) >= vectorCount) |
|||
{ |
|||
TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); |
|||
index += vectorCount; |
|||
} |
|||
|
|||
if (index != length) |
|||
{ |
|||
end.StoreUnsafe(ref destinationRef, length - vectorCount); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Implements element-wise negation for scalar and SIMD inputs.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
private readonly struct NegateOperator<T> : IUnaryOperator<T> |
|||
where T : IUnaryNegationOperators<T, T> |
|||
{ |
|||
/// <inheritdoc />
|
|||
public static bool Vectorizable => true; |
|||
|
|||
/// <inheritdoc />
|
|||
public static T Invoke(T x) => -x; |
|||
|
|||
/// <inheritdoc />
|
|||
public static Vector128<T> Invoke(Vector128<T> x) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
// IEEE-754 negation toggles the sign bit. Expressing that operation explicitly avoids the
|
|||
// subtraction-based ARM64 code generated by .NET 8 for generic vector negation, which loses
|
|||
// the sign when +0F is negated and therefore differs from both scalar and runtime-main behavior.
|
|||
return x ^ Vector128.Create(-0F).As<float, T>(); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
// Double-precision values use the same sign-bit representation, with the sign in bit 63.
|
|||
return x ^ Vector128.Create(-0D).As<double, T>(); |
|||
} |
|||
|
|||
return -x; |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public static Vector256<T> Invoke(Vector256<T> x) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
// Keep the operation bitwise at every width so ARM64 preserves signed zero exactly.
|
|||
return x ^ Vector256.Create(-0F).As<float, T>(); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
return x ^ Vector256.Create(-0D).As<double, T>(); |
|||
} |
|||
|
|||
return -x; |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public static Vector512<T> Invoke(Vector512<T> x) |
|||
{ |
|||
if (typeof(T) == typeof(float)) |
|||
{ |
|||
// Vector512 can be hardware accelerated directly or decomposed by the runtime; the explicit
|
|||
// bit operation provides identical IEEE-754 behavior in either case.
|
|||
return x ^ Vector512.Create(-0F).As<float, T>(); |
|||
} |
|||
|
|||
if (typeof(T) == typeof(double)) |
|||
{ |
|||
return x ^ Vector512.Create(-0D).As<double, T>(); |
|||
} |
|||
|
|||
return -x; |
|||
} |
|||
} |
|||
} |
|||
@ -1,580 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using SixLabors.ImageSharp.Common.Helpers; |
|||
using SixLabors.ImageSharp.Tests.TestUtilities; |
|||
|
|||
namespace SixLabors.ImageSharp.Tests.Common; |
|||
|
|||
public class TensorPrimitivesTests |
|||
{ |
|||
private static readonly int[] SpanLengthValues = |
|||
[ |
|||
0, |
|||
1, |
|||
3, |
|||
4, |
|||
5, |
|||
7, |
|||
8, |
|||
9, |
|||
15, |
|||
16, |
|||
17, |
|||
31, |
|||
32, |
|||
33, |
|||
63, |
|||
64, |
|||
65, |
|||
127, |
|||
128, |
|||
129, |
|||
2048 |
|||
]; |
|||
|
|||
/// <summary>
|
|||
/// Gets lengths that exercise scalar execution, every SIMD width, overlapping tails, and the unrolled loop.
|
|||
/// </summary>
|
|||
public static TheoryData<int> SpanLengths => new(SpanLengthValues); |
|||
|
|||
/// <summary>
|
|||
/// Verifies every compatibility operation while forcing the supported SIMD feature tiers in isolated processes.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void OperationsMatchScalarFormulasAcrossHardwareIntrinsicFeatures() |
|||
=> FeatureTestRunner.RunWithHwIntrinsicsFeature( |
|||
RunOperationsAcrossHardwareIntrinsicFeatures, |
|||
HwIntrinsics.AllowAll |
|||
| HwIntrinsics.DisableAVX512F |
|||
| HwIntrinsics.DisableAVX |
|||
| HwIntrinsics.DisableArm64Sve |
|||
| HwIntrinsics.DisableHWIntrinsic); |
|||
|
|||
/// <summary>
|
|||
/// Verifies that span-to-span operations require equal input lengths before accessing either input.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void AddSpanSpanRejectsMismatchedInputLengths() |
|||
{ |
|||
ArgumentException exception = Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(new int[4], new int[3], new int[4])); |
|||
|
|||
Assert.Null(exception.ParamName); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that every operation rejects a destination that cannot hold all input elements.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void OperationsRejectShortDestinations() |
|||
{ |
|||
int[] integers = new int[4]; |
|||
float[] singles = new float[4]; |
|||
|
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Add<int>(integers, integers, new int[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Add<int>(integers, 1, new int[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Multiply<float>(singles, 2F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Divide<float>(singles, 2F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Max<float>(singles, 2F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Clamp<float>(singles, 0F, 1F, new float[3])).ParamName); |
|||
Assert.Equal("destination", Assert.Throws<ArgumentException>(() => TensorPrimitives_.Negate<float>(singles, new float[3])).ParamName); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that every operation rejects shifted input and destination overlap.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void OperationsRejectShiftedOverlap() |
|||
{ |
|||
int[] integers = new int[5]; |
|||
int[] separateIntegers = new int[4]; |
|||
float[] singles = new float[5]; |
|||
|
|||
// Both inputs need independent validation because either one may alias a shifted destination.
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(integers.AsSpan(0, 4), separateIntegers, integers.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(separateIntegers, integers.AsSpan(0, 4), integers.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Add<int>(integers.AsSpan(0, 4), 1, integers.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Multiply<float>(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Divide<float>(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Max<float>(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Clamp<float>(singles.AsSpan(0, 4), 0F, 1F, singles.AsSpan(1, 4))).ParamName); |
|||
|
|||
Assert.Equal( |
|||
"destination", |
|||
Assert.Throws<ArgumentException>( |
|||
() => TensorPrimitives_.Negate<float>(singles.AsSpan(0, 4), singles.AsSpan(1, 4))).ParamName); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Runs the TensorPrimitives compatibility assertions inside a process configured for one hardware-intrinsic tier.
|
|||
/// </summary>
|
|||
private static void RunOperationsAcrossHardwareIntrinsicFeatures() |
|||
{ |
|||
TensorPrimitivesTests tests = new(); |
|||
|
|||
// Reuse the focused assertions so the remote feature matrix cannot drift from the normal test coverage.
|
|||
foreach (int length in SpanLengthValues) |
|||
{ |
|||
tests.AddByteMatchesScalarFormula(length); |
|||
tests.AddUInt32MatchesScalarFormula(length); |
|||
tests.AddScalarInt32MatchesScalarFormula(length); |
|||
tests.NegateSingleMatchesScalarFormula(length); |
|||
tests.NegateDoubleMatchesScalarFormula(length); |
|||
tests.ClampInt32MatchesScalarFormula(length); |
|||
tests.ClampSingleMatchesRuntimeFormula(length); |
|||
tests.DivideSingleMatchesScalarFormula(length); |
|||
tests.MaxSingleMatchesRuntimeFormula(length); |
|||
tests.MultiplySingleMatchesScalarFormula(length); |
|||
tests.NormalizeMatchesScalarFormula(length); |
|||
} |
|||
|
|||
tests.ClampSinglePreservesRuntimeSpecialValueSemantics(); |
|||
tests.ClampDoublePreservesRuntimeSpecialValueSemantics(); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that byte addition wraps modulo 256 and supports either input as the in-place destination.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void AddByteMatchesScalarFormula(int length) |
|||
{ |
|||
byte[] x = new byte[length]; |
|||
byte[] y = new byte[length]; |
|||
byte[] expected = new byte[length]; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
x[i] = (byte)((i * 23) + 197); |
|||
y[i] = (byte)((i * 41) + 113); |
|||
expected[i] = unchecked((byte)(x[i] + y[i])); |
|||
} |
|||
|
|||
byte[] destination = new byte[length]; |
|||
TensorPrimitives_.Add<byte>(x, y, destination); |
|||
Assert.Equal(expected, destination); |
|||
|
|||
byte[] xInPlace = (byte[])x.Clone(); |
|||
TensorPrimitives_.Add<byte>(xInPlace, y, xInPlace); |
|||
Assert.Equal(expected, xInPlace); |
|||
|
|||
byte[] yInPlace = (byte[])y.Clone(); |
|||
TensorPrimitives_.Add<byte>(x, yInPlace, yInPlace); |
|||
Assert.Equal(expected, yInPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that unsigned integer addition preserves unchecked histogram accumulation semantics.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void AddUInt32MatchesScalarFormula(int length) |
|||
{ |
|||
uint[] x = new uint[length]; |
|||
uint[] y = new uint[length]; |
|||
uint[] expected = new uint[length]; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
x[i] = ((uint)i * 1_234_567U) + 0xF0000000U; |
|||
y[i] = ((uint)i * 7_654_321U) + 0x30000000U; |
|||
expected[i] = unchecked(x[i] + y[i]); |
|||
} |
|||
|
|||
TensorPrimitives_.Add<uint>(x, y, x); |
|||
Assert.Equal(expected, x); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that scalar integer addition produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void AddScalarInt32MatchesScalarFormula(int length) |
|||
{ |
|||
int[] source = new int[length]; |
|||
int[] expected = new int[length]; |
|||
const int addend = 17; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
source[i] = (i * 37) - 200; |
|||
expected[i] = source[i] + addend; |
|||
} |
|||
|
|||
int[] destination = new int[length]; |
|||
TensorPrimitives_.Add(source, addend, destination); |
|||
Assert.Equal(expected, destination); |
|||
|
|||
int[] inPlace = (int[])source.Clone(); |
|||
TensorPrimitives_.Add(inPlace, addend, inPlace); |
|||
Assert.Equal(expected, inPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that floating-point negation preserves the scalar operator's exact bit-level behavior.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void NegateSingleMatchesScalarFormula(int length) |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
-0F, |
|||
0F, |
|||
-1F, |
|||
1F, |
|||
float.NegativeInfinity, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = values[i % values.Length]; |
|||
expected[i] = -source[i]; |
|||
} |
|||
|
|||
float[] destination = new float[length]; |
|||
TensorPrimitives_.Negate<float>(source, destination); |
|||
AssertSingleBitsEqual(expected, destination); |
|||
|
|||
TensorPrimitives_.Negate<float>(source, source); |
|||
AssertSingleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that double-precision negation preserves the scalar operator's exact bit-level behavior.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void NegateDoubleMatchesScalarFormula(int length) |
|||
{ |
|||
double[] values = |
|||
[ |
|||
double.NaN, |
|||
-0D, |
|||
0D, |
|||
-1D, |
|||
1D, |
|||
double.NegativeInfinity, |
|||
double.PositiveInfinity |
|||
]; |
|||
|
|||
double[] source = new double[length]; |
|||
double[] expected = new double[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = values[i % values.Length]; |
|||
expected[i] = -source[i]; |
|||
} |
|||
|
|||
double[] destination = new double[length]; |
|||
TensorPrimitives_.Negate<double>(source, destination); |
|||
AssertDoubleBitsEqual(expected, destination); |
|||
|
|||
TensorPrimitives_.Negate<double>(source, source); |
|||
AssertDoubleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that integer clamping produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void ClampInt32MatchesScalarFormula(int length) |
|||
{ |
|||
int[] source = new int[length]; |
|||
int[] expected = new int[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = ((i * 37) % 401) - 200; |
|||
expected[i] = Math.Clamp(source[i], -73, 91); |
|||
} |
|||
|
|||
int[] destination = new int[length]; |
|||
TensorPrimitives_.Clamp<int>(source, -73, 91, destination); |
|||
Assert.Equal(expected, destination); |
|||
|
|||
int[] inPlace = (int[])source.Clone(); |
|||
TensorPrimitives_.Clamp<int>(inPlace, -73, 91, inPlace); |
|||
Assert.Equal(expected, inPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that floating-point clamping matches the runtime tensor formula for special values and unordered bounds.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void ClampSingleMatchesRuntimeFormula(int length) |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
-0F, |
|||
0F, |
|||
-1F, |
|||
1F, |
|||
float.NegativeInfinity, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = values[i % values.Length]; |
|||
|
|||
// Runtime main follows Min(Max(x, min), max) for vectorizable types, including unordered bounds.
|
|||
expected[i] = float.Min(float.Max(source[i], 2F), -2F); |
|||
} |
|||
|
|||
TensorPrimitives_.Clamp<float>(source, 2F, -2F, source); |
|||
AssertSingleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that single-precision clamping preserves the runtime's signed-zero and NaN behavior.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void ClampSinglePreservesRuntimeSpecialValueSemantics() |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
float.NegativeInfinity, |
|||
-0F, |
|||
0F, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] actual = new float[129]; |
|||
float[] expected = new float[actual.Length]; |
|||
|
|||
for (int i = 0; i < actual.Length; i++) |
|||
{ |
|||
actual[i] = values[i % values.Length]; |
|||
expected[i] = float.Min(float.Max(actual[i], -0F), 0F); |
|||
} |
|||
|
|||
TensorPrimitives_.Clamp<float>(actual, -0F, 0F, actual); |
|||
AssertSingleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that double-precision clamping preserves the runtime's signed-zero and NaN behavior.
|
|||
/// </summary>
|
|||
[Fact] |
|||
public void ClampDoublePreservesRuntimeSpecialValueSemantics() |
|||
{ |
|||
double[] values = |
|||
[ |
|||
double.NaN, |
|||
double.NegativeInfinity, |
|||
-0D, |
|||
0D, |
|||
double.PositiveInfinity |
|||
]; |
|||
|
|||
double[] actual = new double[65]; |
|||
double[] expected = new double[actual.Length]; |
|||
|
|||
for (int i = 0; i < actual.Length; i++) |
|||
{ |
|||
actual[i] = values[i % values.Length]; |
|||
expected[i] = double.Min(double.Max(actual[i], -0D), 0D); |
|||
} |
|||
|
|||
TensorPrimitives_.Clamp<double>(actual, -0D, 0D, actual); |
|||
AssertDoubleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that division produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void DivideSingleMatchesScalarFormula(int length) |
|||
{ |
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = (i - 65.25F) * 1.75F; |
|||
expected[i] = source[i] / 3.25F; |
|||
} |
|||
|
|||
float[] destination = new float[length]; |
|||
TensorPrimitives_.Divide<float>(source, 3.25F, destination); |
|||
AssertSingleBitsEqual(expected, destination); |
|||
|
|||
float[] inPlace = (float[])source.Clone(); |
|||
TensorPrimitives_.Divide<float>(inPlace, 3.25F, inPlace); |
|||
AssertSingleBitsEqual(expected, inPlace); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that maximum selection preserves the runtime's NaN and signed-zero semantics.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void MaxSingleMatchesRuntimeFormula(int length) |
|||
{ |
|||
float[] values = |
|||
[ |
|||
float.NaN, |
|||
float.NegativeInfinity, |
|||
-1F, |
|||
-0F, |
|||
0F, |
|||
1F, |
|||
float.PositiveInfinity |
|||
]; |
|||
|
|||
float[] actual = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < length; i++) |
|||
{ |
|||
actual[i] = values[i % values.Length]; |
|||
expected[i] = float.Max(actual[i], -0F); |
|||
} |
|||
|
|||
TensorPrimitives_.Max<float>(actual, -0F, actual); |
|||
AssertSingleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that multiplication produces identical results for separate and in-place destinations.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void MultiplySingleMatchesScalarFormula(int length) |
|||
{ |
|||
float[] source = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < source.Length; i++) |
|||
{ |
|||
source[i] = (i - 65.25F) * 1.75F; |
|||
expected[i] = source[i] * 0.375F; |
|||
} |
|||
|
|||
float[] destination = new float[length]; |
|||
TensorPrimitives_.Multiply<float>(source, 0.375F, destination); |
|||
AssertSingleBitsEqual(expected, destination); |
|||
|
|||
TensorPrimitives_.Multiply<float>(source, 0.375F, source); |
|||
AssertSingleBitsEqual(expected, source); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Verifies that the normalization compatibility call preserves its element-wise division contract.
|
|||
/// </summary>
|
|||
/// <param name="length">The input length.</param>
|
|||
[Theory] |
|||
[MemberData(nameof(SpanLengths))] |
|||
public void NormalizeMatchesScalarFormula(int length) |
|||
{ |
|||
float[] actual = new float[length]; |
|||
float[] expected = new float[length]; |
|||
|
|||
for (int i = 0; i < actual.Length; i++) |
|||
{ |
|||
actual[i] = (i + 1) * 0.125F; |
|||
expected[i] = actual[i] / 7.5F; |
|||
} |
|||
|
|||
Numerics.Normalize(actual, 7.5F); |
|||
AssertSingleBitsEqual(expected, actual); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Compares floating-point results while preserving signed-zero behavior.
|
|||
/// </summary>
|
|||
/// <param name="expected">The expected values.</param>
|
|||
/// <param name="actual">The actual values.</param>
|
|||
private static void AssertSingleBitsEqual(ReadOnlySpan<float> expected, ReadOnlySpan<float> actual) |
|||
{ |
|||
Assert.Equal(expected.Length, actual.Length); |
|||
|
|||
for (int i = 0; i < expected.Length; i++) |
|||
{ |
|||
if (float.IsNaN(expected[i])) |
|||
{ |
|||
Assert.True(float.IsNaN(actual[i])); |
|||
} |
|||
else |
|||
{ |
|||
Assert.Equal(BitConverter.SingleToInt32Bits(expected[i]), BitConverter.SingleToInt32Bits(actual[i])); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Compares double-precision results while preserving signed-zero behavior.
|
|||
/// </summary>
|
|||
/// <param name="expected">The expected values.</param>
|
|||
/// <param name="actual">The actual values.</param>
|
|||
private static void AssertDoubleBitsEqual(ReadOnlySpan<double> expected, ReadOnlySpan<double> actual) |
|||
{ |
|||
Assert.Equal(expected.Length, actual.Length); |
|||
|
|||
for (int i = 0; i < expected.Length; i++) |
|||
{ |
|||
if (double.IsNaN(expected[i])) |
|||
{ |
|||
Assert.True(double.IsNaN(actual[i])); |
|||
} |
|||
else |
|||
{ |
|||
Assert.Equal(BitConverter.DoubleToInt64Bits(expected[i]), BitConverter.DoubleToInt64Bits(actual[i])); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue