Browse Source

Add TensorPrimitives compatibility implementation

pull/3161/head
James Jackson-South 3 weeks ago
parent
commit
ebe5cc64da
  1. 2
      shared-infrastructure
  2. 73
      src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs
  3. 186
      src/ImageSharp/Common/Helpers/Numerics.cs
  4. 1566
      src/ImageSharp/Common/Helpers/TensorPrimitives.cs
  5. 95
      src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs
  6. 126
      src/ImageSharp/Formats/Png/Filters/UpFilter.cs
  7. 27
      src/ImageSharp/Formats/Webp/AlphaDecoder.cs
  8. 47
      src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs
  9. 65
      tests/ImageSharp.Benchmarks/General/BasicMath/AddSpan.cs
  10. 61
      tests/ImageSharp.Benchmarks/General/BasicMath/NormalizeSpan.cs
  11. 813
      tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssemblyComparison.cs
  12. 1
      tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj
  13. 356
      tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs

2
shared-infrastructure

@ -1 +1 @@
Subproject commit 7ac5703452348d9295db31fc0912c2bd9e419dc9
Subproject commit 74b7f32b8e41fdf8fe2f3eda54fd5a82ebbedfbc

73
src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs

@ -5,9 +5,11 @@ using System.Buffers;
using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.ColorProfiles.Conversion.Icc;
using SixLabors.ImageSharp.ColorProfiles.Icc;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.Metadata.Profiles.Icc;
@ -658,38 +660,8 @@ internal static class ColorProfileConverterExtensionsIcc
private static void ClipNegative(Span<Vector4> source)
{
if (Vector.IsHardwareAccelerated && Vector<float>.IsSupported && Vector<float>.Count >= source.Length * 4)
{
// SIMD loop
int i = 0;
int simdBatchSize = Vector<float>.Count / 4; // Number of Vector4 elements per SIMD batch
for (; i <= source.Length - simdBatchSize; i += simdBatchSize)
{
// Load the vector from source span
Vector<float> v = Unsafe.ReadUnaligned<Vector<float>>(ref Unsafe.As<Vector4, byte>(ref source[i]));
v = Vector.Max(v, Vector<float>.Zero);
// Write the vector to the destination span
Unsafe.WriteUnaligned(ref Unsafe.As<Vector4, byte>(ref source[i]), v);
}
// Scalar fallback for remaining elements
for (; i < source.Length; i++)
{
ref Vector4 s = ref source[i];
s = Vector4.Max(s, Vector4.Zero);
}
}
else
{
// Scalar fallback if SIMD is not supported
for (int i = 0; i < source.Length; i++)
{
ref Vector4 s = ref source[i];
s = Vector4.Max(s, Vector4.Zero);
}
}
Span<float> values = MemoryMarshal.Cast<Vector4, float>(source);
TensorPrimitives_.Max(values, 0F, values);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@ -708,39 +680,10 @@ internal static class ColorProfileConverterExtensionsIcc
private static void LabToLab(Span<Vector4> source, Span<Vector4> destination, [ConstantExpected] float scale)
{
if (Vector.IsHardwareAccelerated && Vector<float>.IsSupported)
{
Vector<float> vScale = new(scale);
int i = 0;
// SIMD loop
int simdBatchSize = Vector<float>.Count / 4; // Number of Vector4 elements per SIMD batch
for (; i <= source.Length - simdBatchSize; i += simdBatchSize)
{
// Load the vector from source span
Vector<float> v = Unsafe.ReadUnaligned<Vector<float>>(ref Unsafe.As<Vector4, byte>(ref source[i]));
// Scale the vector
v *= vScale;
// Write the scaled vector to the destination span
Unsafe.WriteUnaligned(ref Unsafe.As<Vector4, byte>(ref destination[i]), v);
}
// Scalar fallback for remaining elements
for (; i < source.Length; i++)
{
destination[i] = source[i] * scale;
}
}
else
{
// Scalar fallback if SIMD is not supported
for (int i = 0; i < source.Length; i++)
{
destination[i] = source[i] * scale;
}
}
TensorPrimitives_.Multiply(
MemoryMarshal.Cast<Vector4, float>(source),
scale,
MemoryMarshal.Cast<Vector4, float>(destination));
}
private class ConversionParams

186
src/ImageSharp/Common/Helpers/Numerics.cs

@ -329,22 +329,7 @@ internal static class Numerics
/// <param name="max">The maximum inclusive value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Clamp(Span<byte> span, byte min, byte max)
{
Span<byte> remainder = span[ClampReduce(span, min, max)..];
if (remainder.Length > 0)
{
ref byte remainderStart = ref MemoryMarshal.GetReference(remainder);
ref byte remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length);
while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd))
{
remainderStart = Clamp(remainderStart, min, max);
remainderStart = ref Unsafe.Add(ref remainderStart, 1);
}
}
}
=> TensorPrimitives_.Clamp(span, min, max, span);
/// <summary>
/// Clamps the span values to the inclusive range of min and max.
@ -354,22 +339,7 @@ internal static class Numerics
/// <param name="max">The maximum inclusive value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Clamp(Span<uint> span, uint min, uint max)
{
Span<uint> remainder = span[ClampReduce(span, min, max)..];
if (remainder.Length > 0)
{
ref uint remainderStart = ref MemoryMarshal.GetReference(remainder);
ref uint remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length);
while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd))
{
remainderStart = Clamp(remainderStart, min, max);
remainderStart = ref Unsafe.Add(ref remainderStart, 1);
}
}
}
=> TensorPrimitives_.Clamp(span, min, max, span);
/// <summary>
/// Clamps the span values to the inclusive range of min and max.
@ -379,22 +349,7 @@ internal static class Numerics
/// <param name="max">The maximum inclusive value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Clamp(Span<int> span, int min, int max)
{
Span<int> remainder = span[ClampReduce(span, min, max)..];
if (remainder.Length > 0)
{
ref int remainderStart = ref MemoryMarshal.GetReference(remainder);
ref int remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length);
while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd))
{
remainderStart = Clamp(remainderStart, min, max);
remainderStart = ref Unsafe.Add(ref remainderStart, 1);
}
}
}
=> TensorPrimitives_.Clamp(span, min, max, span);
/// <summary>
/// Clamps the span values to the inclusive range of min and max.
@ -404,22 +359,7 @@ internal static class Numerics
/// <param name="max">The maximum inclusive value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Clamp(Span<float> span, float min, float max)
{
Span<float> remainder = span[ClampReduce(span, min, max)..];
if (remainder.Length > 0)
{
ref float remainderStart = ref MemoryMarshal.GetReference(remainder);
ref float remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length);
while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd))
{
remainderStart = Clamp(remainderStart, min, max);
remainderStart = ref Unsafe.Add(ref remainderStart, 1);
}
}
}
=> TensorPrimitives_.Clamp(span, min, max, span);
/// <summary>
/// Clamps the span values to the inclusive range of min and max.
@ -429,87 +369,7 @@ internal static class Numerics
/// <param name="max">The maximum inclusive value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Clamp(Span<double> span, double min, double max)
{
Span<double> remainder = span[ClampReduce(span, min, max)..];
if (remainder.Length > 0)
{
ref double remainderStart = ref MemoryMarshal.GetReference(remainder);
ref double remainderEnd = ref Unsafe.Add(ref remainderStart, (uint)remainder.Length);
while (Unsafe.IsAddressLessThan(ref remainderStart, ref remainderEnd))
{
remainderStart = Clamp(remainderStart, min, max);
remainderStart = ref Unsafe.Add(ref remainderStart, 1);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int ClampReduce<T>(Span<T> span, T min, T max)
where T : unmanaged
{
if (Vector.IsHardwareAccelerated && span.Length >= Vector<T>.Count)
{
int remainder = ModuloP2(span.Length, Vector<T>.Count);
int adjustedCount = span.Length - remainder;
if (adjustedCount > 0)
{
ClampImpl(span[..adjustedCount], min, max);
}
return adjustedCount;
}
return 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ClampImpl<T>(Span<T> span, T min, T max)
where T : unmanaged
{
ref T sRef = ref MemoryMarshal.GetReference(span);
Vector<T> vmin = new(min);
Vector<T> vmax = new(max);
nint n = (nint)(uint)span.Length / Vector<T>.Count;
nint m = Modulo4(n);
nint u = n - m;
ref Vector<T> vs0 = ref Unsafe.As<T, Vector<T>>(ref MemoryMarshal.GetReference(span));
ref Vector<T> vs1 = ref Unsafe.Add(ref vs0, 1);
ref Vector<T> vs2 = ref Unsafe.Add(ref vs0, 2);
ref Vector<T> vs3 = ref Unsafe.Add(ref vs0, 3);
ref Vector<T> vsEnd = ref Unsafe.Add(ref vs0, u);
while (Unsafe.IsAddressLessThan(ref vs0, ref vsEnd))
{
vs0 = Vector.Min(Vector.Max(vmin, vs0), vmax);
vs1 = Vector.Min(Vector.Max(vmin, vs1), vmax);
vs2 = Vector.Min(Vector.Max(vmin, vs2), vmax);
vs3 = Vector.Min(Vector.Max(vmin, vs3), vmax);
vs0 = ref Unsafe.Add(ref vs0, 4);
vs1 = ref Unsafe.Add(ref vs1, 4);
vs2 = ref Unsafe.Add(ref vs2, 4);
vs3 = ref Unsafe.Add(ref vs3, 4);
}
if (m > 0)
{
vs0 = ref vsEnd;
vsEnd = ref Unsafe.Add(ref vsEnd, m);
while (Unsafe.IsAddressLessThan(ref vs0, ref vsEnd))
{
vs0 = Vector.Min(Vector.Max(vmin, vs0), vmax);
vs0 = ref Unsafe.Add(ref vs0, 1);
}
}
}
=> TensorPrimitives_.Clamp(span, min, max, span);
/// <summary>
/// Pre-multiplies the "x", "y", "z" components of a vector by its "w" component leaving the "w" component intact.
@ -1210,39 +1070,5 @@ internal static class Numerics
/// <param name="sum">The sum of the values in <paramref name="span"/>.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Normalize(Span<float> span, float sum)
{
if (Vector256.IsHardwareAccelerated)
{
ref float startRef = ref MemoryMarshal.GetReference(span);
ref float endRef = ref Unsafe.Add(ref startRef, span.Length & ~7);
Vector256<float> sum256 = Vector256.Create(sum);
while (Unsafe.IsAddressLessThan(ref startRef, ref endRef))
{
Unsafe.As<float, Vector256<float>>(ref startRef) /= sum256;
startRef = ref Unsafe.Add(ref startRef, (nuint)8);
}
if ((span.Length & 7) >= 4)
{
Unsafe.As<float, Vector128<float>>(ref startRef) /= sum256.GetLower();
startRef = ref Unsafe.Add(ref startRef, (nuint)4);
}
endRef = ref Unsafe.Add(ref startRef, span.Length & 3);
while (Unsafe.IsAddressLessThan(ref startRef, ref endRef))
{
startRef /= sum;
startRef = ref Unsafe.Add(ref startRef, (nuint)1);
}
}
else
{
for (int i = 0; i < span.Length; i++)
{
span[i] /= sum;
}
}
}
=> TensorPrimitives_.Divide(span, sum, span);
}

1566
src/ImageSharp/Common/Helpers/TensorPrimitives.cs

File diff suppressed because it is too large

95
src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs

@ -5,8 +5,8 @@ using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder;
@ -116,52 +116,7 @@ internal class ComponentProcessor : IDisposable
}
static void SumVertical(Span<float> target, Span<float> source)
{
if (Avx.IsSupported)
{
ref Vector256<float> targetVectorRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(target));
ref Vector256<float> sourceVectorRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(source));
// Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed
DebugGuard.IsTrue(source.Length % 8 == 0, "source must be multiple of 8");
nuint count = source.Vector256Count<float>();
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVectorRef, i) = Avx.Add(Unsafe.Add(ref targetVectorRef, i), Unsafe.Add(ref sourceVectorRef, i));
}
}
else if (AdvSimd.IsSupported)
{
ref Vector128<float> targetVectorRef = ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(target));
ref Vector128<float> sourceVectorRef = ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(source));
// Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed
DebugGuard.IsTrue(source.Length % 8 == 0, "source must be multiple of 8");
nuint count = source.Vector128Count<float>();
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVectorRef, i) = AdvSimd.Add(Unsafe.Add(ref targetVectorRef, i), Unsafe.Add(ref sourceVectorRef, i));
}
}
else
{
ref Vector<float> targetVectorRef = ref Unsafe.As<float, Vector<float>>(ref MemoryMarshal.GetReference(target));
ref Vector<float> sourceVectorRef = ref Unsafe.As<float, Vector<float>>(ref MemoryMarshal.GetReference(source));
nuint count = source.VectorCount<float>();
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVectorRef, i) += Unsafe.Add(ref sourceVectorRef, i);
}
ref float targetRef = ref MemoryMarshal.GetReference(target);
ref float sourceRef = ref MemoryMarshal.GetReference(source);
for (nuint i = count * (uint)Vector<float>.Count; i < (uint)source.Length; i++)
{
Unsafe.Add(ref targetRef, i) += Unsafe.Add(ref sourceRef, i);
}
}
}
=> TensorPrimitives_.Add(target, source, target);
static void SumHorizontal(Span<float> target, int factor)
{
@ -209,50 +164,6 @@ internal class ComponentProcessor : IDisposable
}
static void MultiplyToAverage(Span<float> target, float multiplier)
{
if (Avx.IsSupported)
{
ref Vector256<float> targetVectorRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(target));
// Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed
DebugGuard.IsTrue(target.Length % 8 == 0, "target must be multiple of 8");
nuint count = target.Vector256Count<float>();
Vector256<float> multiplierVector = Vector256.Create(multiplier);
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVectorRef, i) = Avx.Multiply(Unsafe.Add(ref targetVectorRef, i), multiplierVector);
}
}
else if (AdvSimd.IsSupported)
{
ref Vector128<float> targetVectorRef = ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(target));
// Spans are guaranteed to be multiple of 8 so no extra 'remainder' steps are needed
DebugGuard.IsTrue(target.Length % 8 == 0, "target must be multiple of 8");
nuint count = target.Vector128Count<float>();
Vector128<float> multiplierVector = Vector128.Create(multiplier);
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVectorRef, i) = AdvSimd.Multiply(Unsafe.Add(ref targetVectorRef, i), multiplierVector);
}
}
else
{
ref Vector<float> targetVectorRef = ref Unsafe.As<float, Vector<float>>(ref MemoryMarshal.GetReference(target));
nuint count = target.VectorCount<float>();
Vector<float> multiplierVector = new(multiplier);
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVectorRef, i) *= multiplierVector;
}
ref float targetRef = ref MemoryMarshal.GetReference(target);
for (nuint i = count * (uint)Vector<float>.Count; i < (uint)target.Length; i++)
{
Unsafe.Add(ref targetRef, i) *= multiplier;
}
}
}
=> TensorPrimitives_.Multiply(target, multiplier, target);
}
}

126
src/ImageSharp/Formats/Png/Filters/UpFilter.cs

@ -5,8 +5,8 @@ using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Formats.Png.Filters;
@ -27,128 +27,8 @@ internal static class UpFilter
{
DebugGuard.MustBeSameSized<byte>(scanline, previousScanline, nameof(scanline));
if (Avx2.IsSupported)
{
DecodeAvx2(scanline, previousScanline);
}
else if (Sse2.IsSupported)
{
DecodeSse2(scanline, previousScanline);
}
else if (AdvSimd.IsSupported)
{
DecodeArm(scanline, previousScanline);
}
else
{
DecodeScalar(scanline, previousScanline);
}
}
private static void DecodeAvx2(Span<byte> scanline, Span<byte> previousScanline)
{
ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline);
ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline);
// Up(x) + Prior(x)
int rb = scanline.Length;
nuint offset = 1;
while (rb >= Vector256<byte>.Count)
{
ref byte scanRef = ref Unsafe.Add(ref scanBaseRef, offset);
Vector256<byte> prior = Unsafe.As<byte, Vector256<byte>>(ref scanRef);
Vector256<byte> up = Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref prevBaseRef, offset));
Unsafe.As<byte, Vector256<byte>>(ref scanRef) = Avx2.Add(up, prior);
offset += (uint)Vector256<byte>.Count;
rb -= Vector256<byte>.Count;
}
// Handle left over.
for (nuint i = offset; i < (uint)scanline.Length; i++)
{
ref byte scan = ref Unsafe.Add(ref scanBaseRef, offset);
byte above = Unsafe.Add(ref prevBaseRef, offset);
scan = (byte)(scan + above);
offset++;
}
}
private static void DecodeSse2(Span<byte> scanline, Span<byte> previousScanline)
{
ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline);
ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline);
// Up(x) + Prior(x)
int rb = scanline.Length;
nuint offset = 1;
while (rb >= Vector128<byte>.Count)
{
ref byte scanRef = ref Unsafe.Add(ref scanBaseRef, offset);
Vector128<byte> prior = Unsafe.As<byte, Vector128<byte>>(ref scanRef);
Vector128<byte> up = Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref prevBaseRef, offset));
Unsafe.As<byte, Vector128<byte>>(ref scanRef) = Sse2.Add(up, prior);
offset += (uint)Vector128<byte>.Count;
rb -= Vector128<byte>.Count;
}
// Handle left over.
for (nuint i = offset; i < (uint)scanline.Length; i++)
{
ref byte scan = ref Unsafe.Add(ref scanBaseRef, offset);
byte above = Unsafe.Add(ref prevBaseRef, offset);
scan = (byte)(scan + above);
offset++;
}
}
private static void DecodeArm(Span<byte> scanline, Span<byte> previousScanline)
{
ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline);
ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline);
// Up(x) + Prior(x)
int rb = scanline.Length;
nuint offset = 1;
const int bytesPerBatch = 16;
while (rb >= bytesPerBatch)
{
ref byte scanRef = ref Unsafe.Add(ref scanBaseRef, offset);
Vector128<byte> prior = Unsafe.As<byte, Vector128<byte>>(ref scanRef);
Vector128<byte> up = Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref prevBaseRef, offset));
Unsafe.As<byte, Vector128<byte>>(ref scanRef) = AdvSimd.Add(prior, up);
offset += bytesPerBatch;
rb -= bytesPerBatch;
}
// Handle left over.
for (nuint i = offset; i < (uint)scanline.Length; i++)
{
ref byte scan = ref Unsafe.Add(ref scanBaseRef, offset);
byte above = Unsafe.Add(ref prevBaseRef, offset);
scan = (byte)(scan + above);
offset++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void DecodeScalar(Span<byte> scanline, Span<byte> previousScanline)
{
ref byte scanBaseRef = ref MemoryMarshal.GetReference(scanline);
ref byte prevBaseRef = ref MemoryMarshal.GetReference(previousScanline);
// Up(x) + Prior(x)
for (nuint x = 1; x < (uint)scanline.Length; x++)
{
ref byte scan = ref Unsafe.Add(ref scanBaseRef, x);
byte above = Unsafe.Add(ref prevBaseRef, x);
scan = (byte)(scan + above);
}
// The leading filter byte is metadata; every remaining byte is the modulo-256 sum of Raw(x) and Prior(x).
TensorPrimitives_.Add(scanline[1..], previousScanline[1..], scanline[1..]);
}
/// <summary>

27
src/ImageSharp/Formats/Webp/AlphaDecoder.cs

@ -361,34 +361,9 @@ internal class AlphaDecoder : IDisposable
{
HorizontalUnfilter(null, input, dst, width);
}
else if (Vector256.IsHardwareAccelerated)
{
ref byte inputRef = ref MemoryMarshal.GetReference(input);
ref byte prevRef = ref MemoryMarshal.GetReference(prev);
ref byte dstRef = ref MemoryMarshal.GetReference(dst);
nuint i;
int maxPos = width & ~31;
for (i = 0; i < (uint)maxPos; i += 32)
{
Vector256<int> a0 = Unsafe.As<byte, Vector256<int>>(ref Unsafe.Add(ref inputRef, i));
Vector256<int> b0 = Unsafe.As<byte, Vector256<int>>(ref Unsafe.Add(ref prevRef, i));
Vector256<byte> c0 = a0.AsByte() + b0.AsByte();
ref byte outputRef = ref Unsafe.Add(ref dstRef, i);
Unsafe.As<byte, Vector256<byte>>(ref outputRef) = c0;
}
for (; i < (uint)width; i++)
{
Unsafe.Add(ref dstRef, i) = (byte)(Unsafe.Add(ref prevRef, i) + Unsafe.Add(ref inputRef, i));
}
}
else
{
for (int i = 0; i < width; i++)
{
dst[i] = (byte)(prev[i] + input[i]);
}
TensorPrimitives_.Add(input[..width], prev[..width], dst[..width]);
}
}

47
src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs

@ -3,9 +3,7 @@
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Webp.Lossless;
@ -542,48 +540,7 @@ internal abstract unsafe class Vp8LHistogram
DebugGuard.MustBeGreaterThanOrEqualTo(b.Length, count, nameof(b.Length));
DebugGuard.MustBeGreaterThanOrEqualTo(output.Length, count, nameof(output.Length));
if (Avx2.IsSupported && count >= 32)
{
ref uint aRef = ref MemoryMarshal.GetReference(a);
ref uint bRef = ref MemoryMarshal.GetReference(b);
ref uint outputRef = ref MemoryMarshal.GetReference(output);
nuint idx = 0;
do
{
// Load values.
Vector256<uint> a0 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref aRef, idx + 0));
Vector256<uint> a1 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref aRef, idx + 8));
Vector256<uint> a2 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref aRef, idx + 16));
Vector256<uint> a3 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref aRef, idx + 24));
Vector256<uint> b0 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref bRef, idx + 0));
Vector256<uint> b1 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref bRef, idx + 8));
Vector256<uint> b2 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref bRef, idx + 16));
Vector256<uint> b3 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref bRef, idx + 24));
// Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But
// that's ok since the histogram values are less than 1<<28 (max picture count).
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref outputRef, idx + 0)) = Avx2.Add(a0, b0);
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref outputRef, idx + 8)) = Avx2.Add(a1, b1);
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref outputRef, idx + 16)) = Avx2.Add(a2, b2);
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref outputRef, idx + 24)) = Avx2.Add(a3, b3);
idx += 32;
}
while (idx <= (uint)count - 32);
int i = (int)idx;
for (; i < count; i++)
{
output[i] = a[i] + b[i];
}
}
else
{
for (int i = 0; i < count; i++)
{
output[i] = a[i] + b[i];
}
}
TensorPrimitives_.Add(a[..count], b[..count], output[..count]);
}
}

65
tests/ImageSharp.Benchmarks/General/BasicMath/AddSpan.cs

@ -0,0 +1,65 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath;
public class AddSpan
{
private byte[] scalarValues = null!;
private byte[] tensorValues = null!;
private byte[] addends = null!;
/// <summary>
/// Gets or sets the number of values to add.
/// </summary>
[Params(32, 257, 2048)]
public int Length { get; set; }
/// <summary>
/// Creates equivalent deterministic inputs for both implementations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.scalarValues = new byte[this.Length];
this.tensorValues = new byte[this.Length];
this.addends = new byte[this.Length];
for (int i = 0; i < this.Length; i++)
{
byte value = (byte)((i * 17) + 31);
this.scalarValues[i] = value;
this.tensorValues[i] = value;
this.addends[i] = (byte)((i * 29) + 7);
}
}
/// <summary>
/// Adds the values with a scalar loop.
/// </summary>
/// <returns>The first result, which keeps the mutated data observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public byte Scalar()
{
for (int i = 0; i < this.scalarValues.Length; i++)
{
this.scalarValues[i] += this.addends[i];
}
return this.scalarValues[0];
}
/// <summary>
/// Adds the values with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the mutated data observable to the benchmark harness.</returns>
[Benchmark]
public byte TensorPipeline()
{
TensorPrimitives_.Add<byte>(this.tensorValues, this.addends, this.tensorValues);
return this.tensorValues[0];
}
}

61
tests/ImageSharp.Benchmarks/General/BasicMath/NormalizeSpan.cs

@ -0,0 +1,61 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath;
public class NormalizeSpan
{
private float[] scalarValues = null!;
private float[] tensorValues = null!;
/// <summary>
/// Gets or sets the number of values to normalize.
/// </summary>
[Params(7, 32, 257, 2048)]
public int Length { get; set; }
/// <summary>
/// Creates equivalent deterministic inputs for both implementations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.scalarValues = new float[this.Length];
this.tensorValues = new float[this.Length];
for (int i = 0; i < this.scalarValues.Length; i++)
{
float value = ((i * 17) % 251) + 1;
this.scalarValues[i] = value;
this.tensorValues[i] = value;
}
}
/// <summary>
/// Normalizes the values with a scalar loop.
/// </summary>
/// <returns>The first result, which keeps the mutated data observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public float Scalar()
{
for (int i = 0; i < this.scalarValues.Length; i++)
{
this.scalarValues[i] /= 4096F;
}
return this.scalarValues[0];
}
/// <summary>
/// Normalizes the values with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the mutated data observable to the benchmark harness.</returns>
[Benchmark]
public float TensorPipeline()
{
Numerics.Normalize(this.tensorValues, 4096F);
return this.tensorValues[0];
}
}

813
tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssemblyComparison.cs

@ -0,0 +1,813 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath;
#pragma warning disable SA1649 // File name should match first type name
public class TensorPrimitivesJpegMultiplyAssemblyComparison
#pragma warning restore SA1649 // File name should match first type name
{
private readonly float multiplier = -1F;
private float[] legacyValues = null!;
private float[] tensorValues = null!;
/// <summary>
/// Creates equivalent stable inputs for both implementations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.legacyValues = new float[256];
this.tensorValues = new float[256];
for (int i = 0; i < this.legacyValues.Length; i++)
{
float value = ((i * 17) % 251) + 1;
this.legacyValues[i] = value;
this.tensorValues[i] = value;
}
}
/// <summary>
/// Multiplies the row with the retired JPEG AVX pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public float Legacy()
{
LegacyMultiply(this.legacyValues, this.multiplier);
return this.legacyValues[0];
}
/// <summary>
/// Multiplies the row with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float Tensor()
{
TensorPrimitives_.Multiply(this.tensorValues, this.multiplier, this.tensorValues);
return this.tensorValues[0];
}
/// <summary>
/// Reproduces the retired JPEG multiplication loop for assembly comparison.
/// </summary>
/// <param name="target">The row to multiply.</param>
/// <param name="multiplier">The scalar multiplier.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyMultiply(Span<float> target, float multiplier)
{
ref Vector256<float> targetVector = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(target));
nuint count = (uint)target.Length / (uint)Vector256<float>.Count;
Vector256<float> multiplierVector = Vector256.Create(multiplier);
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVector, i) = Avx.Multiply(Unsafe.Add(ref targetVector, i), multiplierVector);
}
}
}
public class TensorPrimitivesNormalizeAssemblyComparison
{
private readonly float divisor = -1F;
private float[] legacyValues = null!;
private float[] tensorValues = null!;
/// <summary>
/// Creates equivalent stable inputs for both implementations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.legacyValues = new float[7];
this.tensorValues = new float[7];
for (int i = 0; i < this.legacyValues.Length; i++)
{
float value = ((i * 17) % 251) + 1;
this.legacyValues[i] = value;
this.tensorValues[i] = value;
}
}
/// <summary>
/// Normalizes the values with the retired fixed-width pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public float Legacy()
{
LegacyNormalize(this.legacyValues, this.divisor);
return this.legacyValues[0];
}
/// <summary>
/// Normalizes the values with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float Tensor()
{
Numerics.Normalize(this.tensorValues, this.divisor);
return this.tensorValues[0];
}
/// <summary>
/// Reproduces the retired normalization loop for assembly comparison.
/// </summary>
/// <param name="span">The values to normalize.</param>
/// <param name="sum">The scalar divisor.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyNormalize(Span<float> span, float sum)
{
ref float start = ref MemoryMarshal.GetReference(span);
ref float vectorEnd = ref Unsafe.Add(ref start, span.Length & ~7);
Vector256<float> sum256 = Vector256.Create(sum);
while (Unsafe.IsAddressLessThan(ref start, ref vectorEnd))
{
Unsafe.As<float, Vector256<float>>(ref start) /= sum256;
start = ref Unsafe.Add(ref start, (nuint)8);
}
if ((span.Length & 7) >= 4)
{
Unsafe.As<float, Vector128<float>>(ref start) /= sum256.GetLower();
start = ref Unsafe.Add(ref start, (nuint)4);
}
ref float end = ref Unsafe.Add(ref start, span.Length & 3);
while (Unsafe.IsAddressLessThan(ref start, ref end))
{
start /= sum;
start = ref Unsafe.Add(ref start, (nuint)1);
}
}
}
public class TensorPrimitivesUInt32AssemblyComparison
{
private uint[] x = null!;
private uint[] y = null!;
private uint[] legacyDestination = null!;
private uint[] tensorDestination = null!;
/// <summary>
/// Creates deterministic histogram inputs and independent destinations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.x = new uint[2048];
this.y = new uint[2048];
this.legacyDestination = new uint[2048];
this.tensorDestination = new uint[2048];
for (int i = 0; i < this.x.Length; i++)
{
this.x[i] = (uint)((i * 17) + 31);
this.y[i] = (uint)((i * 29) + 7);
}
}
/// <summary>
/// Adds histogram bins with the retired four-vector AVX2 pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public uint Legacy()
{
LegacyAdd(this.x, this.y, this.legacyDestination);
return this.legacyDestination[0];
}
/// <summary>
/// Adds histogram bins with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public uint Tensor()
{
TensorPrimitives_.Add<uint>(this.x, this.y, this.tensorDestination);
return this.tensorDestination[0];
}
/// <summary>
/// Reproduces the retired WebP histogram addition loop for assembly comparison.
/// </summary>
/// <param name="x">The first histogram.</param>
/// <param name="y">The second histogram.</param>
/// <param name="destination">The destination receiving the sums.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyAdd(ReadOnlySpan<uint> x, ReadOnlySpan<uint> y, Span<uint> destination)
{
ref uint xRef = ref MemoryMarshal.GetReference(x);
ref uint yRef = ref MemoryMarshal.GetReference(y);
ref uint destinationRef = ref MemoryMarshal.GetReference(destination);
nuint index = 0;
do
{
Vector256<uint> x0 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref xRef, index));
Vector256<uint> x1 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref xRef, index + 8));
Vector256<uint> x2 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref xRef, index + 16));
Vector256<uint> x3 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref xRef, index + 24));
Vector256<uint> y0 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref yRef, index));
Vector256<uint> y1 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref yRef, index + 8));
Vector256<uint> y2 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref yRef, index + 16));
Vector256<uint> y3 = Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref yRef, index + 24));
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref destinationRef, index)) = Avx2.Add(x0, y0);
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref destinationRef, index + 8)) = Avx2.Add(x1, y1);
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref destinationRef, index + 16)) = Avx2.Add(x2, y2);
Unsafe.As<uint, Vector256<uint>>(ref Unsafe.Add(ref destinationRef, index + 24)) = Avx2.Add(x3, y3);
index += 32;
}
while (index <= (uint)x.Length - 32);
for (int i = (int)index; i < x.Length; i++)
{
destination[i] = x[i] + y[i];
}
}
}
public class TensorPrimitivesByteAssemblyComparison
{
private byte[] x = null!;
private byte[] y = null!;
private byte[] legacyDestination = null!;
private byte[] tensorDestination = null!;
/// <summary>
/// Creates deterministic byte inputs and independent destinations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.x = new byte[2048];
this.y = new byte[2048];
this.legacyDestination = new byte[2048];
this.tensorDestination = new byte[2048];
for (int i = 0; i < this.x.Length; i++)
{
this.x[i] = (byte)((i * 17) + 31);
this.y[i] = (byte)((i * 29) + 7);
}
}
/// <summary>
/// Adds bytes with the retired WebP AVX2 pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public byte Legacy()
{
LegacyAdd(this.x, this.y, this.legacyDestination);
return this.legacyDestination[0];
}
/// <summary>
/// Adds bytes with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public byte Tensor()
{
TensorPrimitives_.Add<byte>(this.x, this.y, this.tensorDestination);
return this.tensorDestination[0];
}
/// <summary>
/// Reproduces the retired WebP byte addition loop for assembly comparison.
/// </summary>
/// <param name="x">The first input.</param>
/// <param name="y">The second input.</param>
/// <param name="destination">The destination receiving modulo-256 sums.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyAdd(ReadOnlySpan<byte> x, ReadOnlySpan<byte> y, Span<byte> destination)
{
ref byte xRef = ref MemoryMarshal.GetReference(x);
ref byte yRef = ref MemoryMarshal.GetReference(y);
ref byte destinationRef = ref MemoryMarshal.GetReference(destination);
nuint i;
int maxPosition = x.Length & ~31;
for (i = 0; i < (uint)maxPosition; i += 32)
{
Vector256<int> x0 = Unsafe.As<byte, Vector256<int>>(ref Unsafe.Add(ref xRef, i));
Vector256<int> y0 = Unsafe.As<byte, Vector256<int>>(ref Unsafe.Add(ref yRef, i));
Vector256<byte> result = x0.AsByte() + y0.AsByte();
Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref destinationRef, i)) = result;
}
for (; i < (uint)x.Length; i++)
{
Unsafe.Add(ref destinationRef, i) = (byte)(Unsafe.Add(ref xRef, i) + Unsafe.Add(ref yRef, i));
}
}
}
public class TensorPrimitivesSingleAddAssemblyComparison
{
private float[] legacyTarget = null!;
private float[] tensorTarget = null!;
private float[] source = null!;
/// <summary>
/// Creates deterministic JPEG row inputs.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.legacyTarget = new float[2048];
this.tensorTarget = new float[2048];
this.source = new float[2048];
for (int i = 0; i < this.source.Length; i++)
{
float value = ((i * 17) % 251) + 1;
this.legacyTarget[i] = value;
this.tensorTarget[i] = value;
this.source[i] = ((i * 29) % 31) - 15;
}
}
/// <summary>
/// Adds JPEG row values with the retired AVX pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public float Legacy()
{
LegacyAdd(this.legacyTarget, this.source);
return this.legacyTarget[0];
}
/// <summary>
/// Adds JPEG row values with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float Tensor()
{
TensorPrimitives_.Add<float>(this.tensorTarget, this.source, this.tensorTarget);
return this.tensorTarget[0];
}
/// <summary>
/// Reproduces the retired JPEG row addition loop for assembly comparison.
/// </summary>
/// <param name="target">The destination row.</param>
/// <param name="source">The row added to the destination.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyAdd(Span<float> target, ReadOnlySpan<float> source)
{
ref Vector256<float> targetVector = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(target));
ref Vector256<float> sourceVector = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(source));
nuint count = (uint)source.Length / (uint)Vector256<float>.Count;
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref targetVector, i) = Avx.Add(Unsafe.Add(ref targetVector, i), Unsafe.Add(ref sourceVector, i));
}
}
}
[GenericTypeArguments(typeof(byte))]
[GenericTypeArguments(typeof(uint))]
[GenericTypeArguments(typeof(int))]
[GenericTypeArguments(typeof(float))]
[GenericTypeArguments(typeof(double))]
public class TensorPrimitivesClampAssemblyComparison<T>
where T : unmanaged, INumber<T>
{
private T[] legacyValues = null!;
private T[] tensorValues = null!;
private T min;
private T max;
/// <summary>
/// Creates deterministic clamp inputs for the current element type.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.legacyValues = new T[2048];
this.tensorValues = new T[2048];
this.min = T.CreateTruncating(64);
this.max = T.CreateTruncating(128);
for (int i = 0; i < this.legacyValues.Length; i++)
{
T value = T.CreateTruncating((i * 31) % 257);
this.legacyValues[i] = value;
this.tensorValues[i] = value;
}
}
/// <summary>
/// Clamps values with the retired <see cref="Vector{T}"/> pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public T Legacy()
{
LegacyClamp(this.legacyValues, this.min, this.max);
return this.legacyValues[0];
}
/// <summary>
/// Clamps values with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public T Tensor()
{
TensorPrimitives_.Clamp(this.tensorValues, this.min, this.max, this.tensorValues);
return this.tensorValues[0];
}
/// <summary>
/// Reproduces the retired clamp pipeline for assembly comparison.
/// </summary>
/// <param name="span">The values to clamp.</param>
/// <param name="min">The inclusive lower bound.</param>
/// <param name="max">The inclusive upper bound.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyClamp(Span<T> span, T min, T max)
{
int remainder = Numerics.ModuloP2(span.Length, Vector<T>.Count);
int adjustedCount = span.Length - remainder;
if (adjustedCount > 0)
{
Vector<T> vectorMin = new(min);
Vector<T> vectorMax = new(max);
nint vectorCount = (nint)(uint)adjustedCount / Vector<T>.Count;
nint remainingVectors = Numerics.Modulo4(vectorCount);
nint unrolledVectors = vectorCount - remainingVectors;
ref Vector<T> current0 = ref Unsafe.As<T, Vector<T>>(ref MemoryMarshal.GetReference(span));
ref Vector<T> current1 = ref Unsafe.Add(ref current0, 1);
ref Vector<T> current2 = ref Unsafe.Add(ref current0, 2);
ref Vector<T> current3 = ref Unsafe.Add(ref current0, 3);
ref Vector<T> end = ref Unsafe.Add(ref current0, unrolledVectors);
while (Unsafe.IsAddressLessThan(ref current0, ref end))
{
current0 = Vector.Min(Vector.Max(vectorMin, current0), vectorMax);
current1 = Vector.Min(Vector.Max(vectorMin, current1), vectorMax);
current2 = Vector.Min(Vector.Max(vectorMin, current2), vectorMax);
current3 = Vector.Min(Vector.Max(vectorMin, current3), vectorMax);
current0 = ref Unsafe.Add(ref current0, 4);
current1 = ref Unsafe.Add(ref current1, 4);
current2 = ref Unsafe.Add(ref current2, 4);
current3 = ref Unsafe.Add(ref current3, 4);
}
if (remainingVectors > 0)
{
current0 = ref end;
end = ref Unsafe.Add(ref end, remainingVectors);
while (Unsafe.IsAddressLessThan(ref current0, ref end))
{
current0 = Vector.Min(Vector.Max(vectorMin, current0), vectorMax);
current0 = ref Unsafe.Add(ref current0, 1);
}
}
}
for (int i = adjustedCount; i < span.Length; i++)
{
T value = span[i];
span[i] = value > max ? max : value < min ? min : value;
}
}
}
public class TensorPrimitivesIccMaxAssemblyComparison
{
private Vector4[] legacyValues = null!;
private Vector4[] tensorValues = null!;
/// <summary>
/// Creates deterministic ICC values containing positive and negative channels.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.legacyValues = new Vector4[512];
this.tensorValues = new Vector4[512];
for (int i = 0; i < this.legacyValues.Length; i++)
{
float value = ((i * 17) % 251) - 125;
Vector4 vector = new(value, value + 1, value - 1, value + 2);
this.legacyValues[i] = vector;
this.tensorValues[i] = vector;
}
}
/// <summary>
/// Clips negative channels with the retired ICC pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public float Legacy()
{
for (int i = 0; i < this.legacyValues.Length; i++)
{
this.legacyValues[i] = Vector4.Max(this.legacyValues[i], Vector4.Zero);
}
return this.legacyValues[0].X;
}
/// <summary>
/// Clips negative channels with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float Tensor()
{
Span<float> values = MemoryMarshal.Cast<Vector4, float>(this.tensorValues.AsSpan());
TensorPrimitives_.Max(values, 0F, values);
return values[0];
}
}
public class TensorPrimitivesIccMultiplyAssemblyComparison
{
private readonly float multiplier = 65280F / 65535F;
private Vector4[] source = null!;
private Vector4[] legacyDestination = null!;
private Vector4[] tensorDestination = null!;
/// <summary>
/// Creates deterministic ICC inputs and independent destinations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.source = new Vector4[512];
this.legacyDestination = new Vector4[512];
this.tensorDestination = new Vector4[512];
for (int i = 0; i < this.source.Length; i++)
{
float value = ((i * 17) % 251) + 1;
this.source[i] = new Vector4(value, value + 1, value + 2, value + 3);
}
}
/// <summary>
/// Multiplies ICC channels with the retired <see cref="Vector{T}"/> pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public float Legacy()
{
Span<float> source = MemoryMarshal.Cast<Vector4, float>(this.source.AsSpan());
Span<float> destination = MemoryMarshal.Cast<Vector4, float>(this.legacyDestination.AsSpan());
ref Vector<float> sourceVector = ref Unsafe.As<float, Vector<float>>(ref MemoryMarshal.GetReference(source));
ref Vector<float> destinationVector = ref Unsafe.As<float, Vector<float>>(ref MemoryMarshal.GetReference(destination));
Vector<float> scale = new(this.multiplier);
nuint count = (uint)source.Length / (uint)Vector<float>.Count;
for (nuint i = 0; i < count; i++)
{
Unsafe.Add(ref destinationVector, i) = Unsafe.Add(ref sourceVector, i) * scale;
}
return destination[0];
}
/// <summary>
/// Multiplies ICC channels with the tensor compatibility pipeline.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float Tensor()
{
Span<float> source = MemoryMarshal.Cast<Vector4, float>(this.source.AsSpan());
Span<float> destination = MemoryMarshal.Cast<Vector4, float>(this.tensorDestination.AsSpan());
TensorPrimitives_.Multiply(source, this.multiplier, destination);
return destination[0];
}
}
#if NET10_0_OR_GREATER
[GenericTypeArguments(typeof(byte))]
[GenericTypeArguments(typeof(uint))]
[GenericTypeArguments(typeof(float))]
public class TensorPrimitivesRuntimeAddAssemblyComparison<T>
where T : unmanaged, INumber<T>
{
private T[] x = null!;
private T[] y = null!;
private T[] compatibilityDestination = null!;
private T[] runtimeDestination = null!;
/// <summary>
/// Creates deterministic inputs and independent destinations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.x = new T[2048];
this.y = new T[2048];
this.compatibilityDestination = new T[2048];
this.runtimeDestination = new T[2048];
for (int i = 0; i < this.x.Length; i++)
{
this.x[i] = T.CreateTruncating((i * 17) + 31);
this.y[i] = T.CreateTruncating((i * 29) + 7);
}
}
/// <summary>
/// Adds values with the compatibility implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public T Compatibility()
{
TensorPrimitives_.Add<T>(this.x, this.y, this.compatibilityDestination);
return this.compatibilityDestination[0];
}
/// <summary>
/// Adds values with the .NET runtime implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public T Runtime()
{
System.Numerics.Tensors.TensorPrimitives.Add<T>(this.x, this.y, this.runtimeDestination);
return this.runtimeDestination[0];
}
}
[GenericTypeArguments(typeof(byte))]
[GenericTypeArguments(typeof(uint))]
[GenericTypeArguments(typeof(int))]
[GenericTypeArguments(typeof(float))]
[GenericTypeArguments(typeof(double))]
public class TensorPrimitivesRuntimeClampAssemblyComparison<T>
where T : unmanaged, INumber<T>
{
private T[] compatibilityValues = null!;
private T[] runtimeValues = null!;
private T min;
private T max;
/// <summary>
/// Creates deterministic inputs for both implementations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.compatibilityValues = new T[2048];
this.runtimeValues = new T[2048];
this.min = T.CreateTruncating(64);
this.max = T.CreateTruncating(128);
for (int i = 0; i < this.compatibilityValues.Length; i++)
{
T value = T.CreateTruncating((i * 31) % 257);
this.compatibilityValues[i] = value;
this.runtimeValues[i] = value;
}
}
/// <summary>
/// Clamps values with the compatibility implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark(Baseline = true)]
public T Compatibility()
{
TensorPrimitives_.Clamp(this.compatibilityValues, this.min, this.max, this.compatibilityValues);
return this.compatibilityValues[0];
}
/// <summary>
/// Clamps values with the .NET runtime implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public T Runtime()
{
System.Numerics.Tensors.TensorPrimitives.Clamp(this.runtimeValues, this.min, this.max, this.runtimeValues);
return this.runtimeValues[0];
}
}
public class TensorPrimitivesRuntimeSingleScalarAssemblyComparison
{
private readonly float scalar = -1F;
private float[] compatibilityValues = null!;
private float[] runtimeValues = null!;
/// <summary>
/// Creates equivalent stable inputs for both implementations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.compatibilityValues = new float[2048];
this.runtimeValues = new float[2048];
for (int i = 0; i < this.compatibilityValues.Length; i++)
{
float value = ((i * 17) % 251) + 1;
this.compatibilityValues[i] = value;
this.runtimeValues[i] = value;
}
}
/// <summary>
/// Divides values with the compatibility implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float CompatibilityDivide()
{
TensorPrimitives_.Divide(this.compatibilityValues, this.scalar, this.compatibilityValues);
return this.compatibilityValues[0];
}
/// <summary>
/// Divides values with the .NET runtime implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float RuntimeDivide()
{
System.Numerics.Tensors.TensorPrimitives.Divide(this.runtimeValues, this.scalar, this.runtimeValues);
return this.runtimeValues[0];
}
/// <summary>
/// Computes maximum values with the compatibility implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float CompatibilityMax()
{
TensorPrimitives_.Max(this.compatibilityValues, 0F, this.compatibilityValues);
return this.compatibilityValues[0];
}
/// <summary>
/// Computes maximum values with the .NET runtime implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float RuntimeMax()
{
System.Numerics.Tensors.TensorPrimitives.Max(this.runtimeValues, 0F, this.runtimeValues);
return this.runtimeValues[0];
}
/// <summary>
/// Multiplies values with the compatibility implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float CompatibilityMultiply()
{
TensorPrimitives_.Multiply(this.compatibilityValues, this.scalar, this.compatibilityValues);
return this.compatibilityValues[0];
}
/// <summary>
/// Multiplies values with the .NET runtime implementation.
/// </summary>
/// <returns>The first result, which keeps the writes observable to the benchmark harness.</returns>
[Benchmark]
public float RuntimeMultiply()
{
System.Numerics.Tensors.TensorPrimitives.Multiply(this.runtimeValues, this.scalar, this.runtimeValues);
return this.runtimeValues[0];
}
}
#endif

1
tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj

@ -69,6 +69,7 @@
<PackageReference Include="SharpZipLib" />
<PackageReference Include="SkiaSharp" />
<PackageReference Include="System.Drawing.Common" />
<PackageReference Include="System.Numerics.Tensors" Version="10.0.0" Condition="'$(TargetFramework)' == 'net10.0'" />
</ItemGroup>
<!-- Exclude benchmarks using internals, in case of unsigned benchmark execution: -->

356
tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs

@ -0,0 +1,356 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Tests.Common;
public class TensorPrimitivesTests
{
/// <summary>
/// Gets lengths that exercise scalar execution, every SIMD width, overlapping tails, and the unrolled loop.
/// </summary>
public static TheoryData<int> SpanLengths => new()
{
0,
1,
3,
4,
5,
7,
8,
9,
15,
16,
17,
31,
32,
33,
63,
64,
65,
127,
128,
129,
2048
};
/// <summary>
/// Verifies that byte addition wraps modulo 256 and supports either input as the in-place destination.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void AddByteMatchesScalarFormula(int length)
{
byte[] x = new byte[length];
byte[] y = new byte[length];
byte[] expected = new byte[length];
for (int i = 0; i < length; i++)
{
x[i] = (byte)((i * 23) + 197);
y[i] = (byte)((i * 41) + 113);
expected[i] = unchecked((byte)(x[i] + y[i]));
}
byte[] destination = new byte[length];
TensorPrimitives_.Add<byte>(x, y, destination);
Assert.Equal(expected, destination);
byte[] xInPlace = (byte[])x.Clone();
TensorPrimitives_.Add<byte>(xInPlace, y, xInPlace);
Assert.Equal(expected, xInPlace);
byte[] yInPlace = (byte[])y.Clone();
TensorPrimitives_.Add<byte>(x, yInPlace, yInPlace);
Assert.Equal(expected, yInPlace);
}
/// <summary>
/// Verifies that unsigned integer addition preserves unchecked histogram accumulation semantics.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void AddUInt32MatchesScalarFormula(int length)
{
uint[] x = new uint[length];
uint[] y = new uint[length];
uint[] expected = new uint[length];
for (int i = 0; i < length; i++)
{
x[i] = ((uint)i * 1_234_567U) + 0xF0000000U;
y[i] = ((uint)i * 7_654_321U) + 0x30000000U;
expected[i] = unchecked(x[i] + y[i]);
}
TensorPrimitives_.Add<uint>(x, y, x);
Assert.Equal(expected, x);
}
/// <summary>
/// Verifies that integer clamping produces identical results for separate and in-place destinations.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void ClampInt32MatchesScalarFormula(int length)
{
int[] source = new int[length];
int[] expected = new int[length];
for (int i = 0; i < source.Length; i++)
{
source[i] = ((i * 37) % 401) - 200;
expected[i] = Math.Clamp(source[i], -73, 91);
}
int[] destination = new int[length];
TensorPrimitives_.Clamp<int>(source, -73, 91, destination);
Assert.Equal(expected, destination);
int[] inPlace = (int[])source.Clone();
TensorPrimitives_.Clamp<int>(inPlace, -73, 91, inPlace);
Assert.Equal(expected, inPlace);
}
/// <summary>
/// Verifies that floating-point clamping matches the runtime tensor formula for special values and unordered bounds.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void ClampSingleMatchesRuntimeFormula(int length)
{
float[] values =
{
float.NaN,
-0F,
0F,
-1F,
1F,
float.NegativeInfinity,
float.PositiveInfinity
};
float[] source = new float[length];
float[] expected = new float[length];
for (int i = 0; i < source.Length; i++)
{
source[i] = values[i % values.Length];
// Runtime main follows Min(Max(x, min), max) for vectorizable types, including unordered bounds.
expected[i] = float.Min(float.Max(source[i], 2F), -2F);
}
TensorPrimitives_.Clamp<float>(source, 2F, -2F, source);
AssertSingleBitsEqual(expected, source);
}
/// <summary>
/// Verifies that single-precision clamping preserves the runtime's signed-zero and NaN behavior.
/// </summary>
[Fact]
public void ClampSinglePreservesRuntimeSpecialValueSemantics()
{
float[] values =
{
float.NaN,
float.NegativeInfinity,
-0F,
0F,
float.PositiveInfinity
};
float[] actual = new float[129];
float[] expected = new float[actual.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = values[i % values.Length];
expected[i] = float.Min(float.Max(actual[i], -0F), 0F);
}
TensorPrimitives_.Clamp<float>(actual, -0F, 0F, actual);
AssertSingleBitsEqual(expected, actual);
}
/// <summary>
/// Verifies that double-precision clamping preserves the runtime's signed-zero and NaN behavior.
/// </summary>
[Fact]
public void ClampDoublePreservesRuntimeSpecialValueSemantics()
{
double[] values =
{
double.NaN,
double.NegativeInfinity,
-0D,
0D,
double.PositiveInfinity
};
double[] actual = new double[65];
double[] expected = new double[actual.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = values[i % values.Length];
expected[i] = double.Min(double.Max(actual[i], -0D), 0D);
}
TensorPrimitives_.Clamp<double>(actual, -0D, 0D, actual);
AssertDoubleBitsEqual(expected, actual);
}
/// <summary>
/// Verifies that division produces identical results for separate and in-place destinations.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void DivideSingleMatchesScalarFormula(int length)
{
float[] source = new float[length];
float[] expected = new float[length];
for (int i = 0; i < source.Length; i++)
{
source[i] = (i - 65.25F) * 1.75F;
expected[i] = source[i] / 3.25F;
}
float[] destination = new float[length];
TensorPrimitives_.Divide<float>(source, 3.25F, destination);
AssertSingleBitsEqual(expected, destination);
float[] inPlace = (float[])source.Clone();
TensorPrimitives_.Divide<float>(inPlace, 3.25F, inPlace);
AssertSingleBitsEqual(expected, inPlace);
}
/// <summary>
/// Verifies that maximum selection preserves the runtime's NaN and signed-zero semantics.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void MaxSingleMatchesRuntimeFormula(int length)
{
float[] values =
{
float.NaN,
float.NegativeInfinity,
-1F,
-0F,
0F,
1F,
float.PositiveInfinity
};
float[] actual = new float[length];
float[] expected = new float[length];
for (int i = 0; i < length; i++)
{
actual[i] = values[i % values.Length];
expected[i] = float.Max(actual[i], -0F);
}
TensorPrimitives_.Max<float>(actual, -0F, actual);
AssertSingleBitsEqual(expected, actual);
}
/// <summary>
/// Verifies that multiplication produces identical results for separate and in-place destinations.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void MultiplySingleMatchesScalarFormula(int length)
{
float[] source = new float[length];
float[] expected = new float[length];
for (int i = 0; i < source.Length; i++)
{
source[i] = (i - 65.25F) * 1.75F;
expected[i] = source[i] * 0.375F;
}
float[] destination = new float[length];
TensorPrimitives_.Multiply<float>(source, 0.375F, destination);
AssertSingleBitsEqual(expected, destination);
TensorPrimitives_.Multiply<float>(source, 0.375F, source);
AssertSingleBitsEqual(expected, source);
}
/// <summary>
/// Verifies that the normalization compatibility call preserves its element-wise division contract.
/// </summary>
/// <param name="length">The input length.</param>
[Theory]
[MemberData(nameof(SpanLengths))]
public void NormalizeMatchesScalarFormula(int length)
{
float[] actual = new float[length];
float[] expected = new float[length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = (i + 1) * 0.125F;
expected[i] = actual[i] / 7.5F;
}
Numerics.Normalize(actual, 7.5F);
AssertSingleBitsEqual(expected, actual);
}
/// <summary>
/// Compares floating-point results while preserving signed-zero behavior.
/// </summary>
/// <param name="expected">The expected values.</param>
/// <param name="actual">The actual values.</param>
private static void AssertSingleBitsEqual(ReadOnlySpan<float> expected, ReadOnlySpan<float> actual)
{
Assert.Equal(expected.Length, actual.Length);
for (int i = 0; i < expected.Length; i++)
{
if (float.IsNaN(expected[i]))
{
Assert.True(float.IsNaN(actual[i]));
}
else
{
Assert.Equal(BitConverter.SingleToInt32Bits(expected[i]), BitConverter.SingleToInt32Bits(actual[i]));
}
}
}
/// <summary>
/// Compares double-precision results while preserving signed-zero behavior.
/// </summary>
/// <param name="expected">The expected values.</param>
/// <param name="actual">The actual values.</param>
private static void AssertDoubleBitsEqual(ReadOnlySpan<double> expected, ReadOnlySpan<double> actual)
{
Assert.Equal(expected.Length, actual.Length);
for (int i = 0; i < expected.Length; i++)
{
if (double.IsNaN(expected[i]))
{
Assert.True(double.IsNaN(actual[i]));
}
else
{
Assert.Equal(BitConverter.DoubleToInt64Bits(expected[i]), BitConverter.DoubleToInt64Bits(actual[i]));
}
}
}
}
Loading…
Cancel
Save