diff --git a/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs b/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs index 7f08a7a9b..424f4ec37 100644 --- a/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs +++ b/src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Numerics; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -663,7 +664,7 @@ internal static class ColorProfileConverterExtensionsIcc // Vector4 values are contiguous floats, so flattening preserves the component order // while allowing one shared tensor traversal to process every channel and SIMD tail. Span values = MemoryMarshal.Cast(source); - TensorPrimitives_.Max(values, 0F, values); + TensorPrimitives.Max(values, 0F, values); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -684,7 +685,7 @@ internal static class ColorProfileConverterExtensionsIcc { // Reinterpreting both spans exposes all four components to one multiplication traversal; // the source and destination retain their original Vector4 boundaries after the operation. - TensorPrimitives_.Multiply(MemoryMarshal.Cast(source), scale, MemoryMarshal.Cast(destination)); + TensorPrimitives.Multiply(MemoryMarshal.Cast(source), scale, MemoryMarshal.Cast(destination)); } private class ConversionParams diff --git a/src/ImageSharp/Common/Helpers/Numerics.cs b/src/ImageSharp/Common/Helpers/Numerics.cs index 9be6089dc..e5a6b4549 100644 --- a/src/ImageSharp/Common/Helpers/Numerics.cs +++ b/src/ImageSharp/Common/Helpers/Numerics.cs @@ -1,7 +1,8 @@ -// Copyright (c) Six Labors. +// Copyright (c) Six Labors. // Licensed under the Six Labors Split License. using System.Numerics; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -329,7 +330,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, byte min, byte max) - => TensorPrimitives_.Clamp(span, min, max, span); + => TensorPrimitives.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -339,7 +340,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, uint min, uint max) - => TensorPrimitives_.Clamp(span, min, max, span); + => TensorPrimitives.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -349,7 +350,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, int min, int max) - => TensorPrimitives_.Clamp(span, min, max, span); + => TensorPrimitives.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -359,7 +360,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, float min, float max) - => TensorPrimitives_.Clamp(span, min, max, span); + => TensorPrimitives.Clamp(span, min, max, span); /// /// Clamps the span values to the inclusive range of min and max. @@ -369,7 +370,7 @@ internal static class Numerics /// The maximum inclusive value. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Clamp(Span span, double min, double max) - => TensorPrimitives_.Clamp(span, min, max, span); + => TensorPrimitives.Clamp(span, min, max, span); /// /// Pre-multiplies the "x", "y", "z" components of a vector by its "w" component leaving the "w" component intact. @@ -1032,13 +1033,4 @@ internal static class Numerics public static nuint Vector512Count(int length) where TVector : struct => (uint)length / (uint)Vector512.Count; - - /// - /// Normalizes the values in a given . - /// - /// The sequence of values to normalize. - /// The sum of the values in . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Normalize(Span span, float sum) - => TensorPrimitives_.Divide(span, sum, span); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs index 2c80d8f57..54fe466ea 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs @@ -47,7 +47,7 @@ internal readonly struct WXYZPad3Shuffle4 : IPad3Shuffle4 // Each four-byte group is an XYZW pixel with opaque W. Selecting [3, 0, 1, 2] // produces WXYZ, and offsets 4, 8, and 12 repeat that rotation for the next pixels. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14)); } /// @@ -69,7 +69,7 @@ internal readonly struct WZYXPad3Shuffle4 : IPad3Shuffle4 // Each four-byte group is an XYZW pixel with opaque W. Selecting [3, 2, 1, 0] // produces WZYX, and offsets 4, 8, and 12 repeat that reversal for the next pixels. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); } /// @@ -91,5 +91,5 @@ internal readonly struct ZYXWPad3Shuffle4 : IPad3Shuffle4 // Each four-byte group is an XYZW pixel with opaque W. Selecting [2, 1, 0, 3] // exchanges X and Z to produce ZYXW, with offsets 4, 8, and 12 covering the next pixels. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs index 7de8af4e4..a7f198b35 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs @@ -34,5 +34,5 @@ internal readonly struct ZYXShuffle3 : IShuffle3 // Each four-byte group is a temporary XYZW pixel created by the shuffle pipeline. // Selecting [2, 1, 0, 3] produces ZYXW, and offsets 4, 8, and 12 repeat that // permutation for the next pixels. The pipeline subsequently discards every W byte. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs index 713de342f..0bf4a2016 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs @@ -69,7 +69,7 @@ internal readonly struct WXYZShuffle4 : IShuffle4 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Invoke(Vector128 source) - => Vector128_.ShuffleNative(source, CreateLaneMask()); + => Vector128.ShuffleNative(source, CreateLaneMask()); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -85,7 +85,7 @@ internal readonly struct WXYZShuffle4 : IShuffle4 public static Vector512 Invoke(Vector512 source) // Expand the four-pixel lane permutation across all four 128-bit lanes. - => Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); + => Vector512.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); /// /// Creates the indices that rotate each XYZW pixel to WXYZ within one 128-bit lane. @@ -115,7 +115,7 @@ internal readonly struct WZYXShuffle4 : IShuffle4 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Invoke(Vector128 source) - => Vector128_.ShuffleNative(source, CreateLaneMask()); + => Vector128.ShuffleNative(source, CreateLaneMask()); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -131,7 +131,7 @@ internal readonly struct WZYXShuffle4 : IShuffle4 public static Vector512 Invoke(Vector512 source) // Expand the four-pixel lane permutation across all four 128-bit lanes. - => Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); + => Vector512.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); /// /// Creates the indices that reverse each XYZW pixel to WZYX within one 128-bit lane. @@ -161,7 +161,7 @@ internal readonly struct YZWXShuffle4 : IShuffle4 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Invoke(Vector128 source) - => Vector128_.ShuffleNative(source, CreateLaneMask()); + => Vector128.ShuffleNative(source, CreateLaneMask()); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -177,7 +177,7 @@ internal readonly struct YZWXShuffle4 : IShuffle4 public static Vector512 Invoke(Vector512 source) // Expand the four-pixel lane permutation across all four 128-bit lanes. - => Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); + => Vector512.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); /// /// Creates the indices that rotate each XYZW pixel to YZWX within one 128-bit lane. @@ -209,7 +209,7 @@ internal readonly struct ZYXWShuffle4 : IShuffle4 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Invoke(Vector128 source) - => Vector128_.ShuffleNative(source, CreateLaneMask()); + => Vector128.ShuffleNative(source, CreateLaneMask()); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -225,7 +225,7 @@ internal readonly struct ZYXWShuffle4 : IShuffle4 public static Vector512 Invoke(Vector512 source) // Expand the four-pixel lane permutation across all four 128-bit lanes. - => Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); + => Vector512.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); /// /// Creates the indices that exchange X and Z in each XYZW pixel within one 128-bit lane. @@ -257,7 +257,7 @@ internal readonly struct XWZYShuffle4 : IShuffle4 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Invoke(Vector128 source) - => Vector128_.ShuffleNative(source, CreateLaneMask()); + => Vector128.ShuffleNative(source, CreateLaneMask()); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -273,7 +273,7 @@ internal readonly struct XWZYShuffle4 : IShuffle4 public static Vector512 Invoke(Vector512 source) // Expand the four-pixel lane permutation across all four 128-bit lanes. - => Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); + => Vector512.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask())); /// /// Creates the indices that exchange Y and W in each XYZW pixel within one 128-bit lane. diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs index 8f32a5a6c..ab5cdfbce 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs @@ -49,7 +49,7 @@ internal readonly struct YZWXShuffle4Slice3 : IShuffle4Slice3 // Each four-byte group is an XYZW pixel. Selecting [1, 2, 3, 0] produces // YZWX, and offsets 4, 8, and 12 repeat that rotation for the next pixels. // The surrounding pipeline subsequently removes every fourth byte. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12)); } /// @@ -72,7 +72,7 @@ internal readonly struct WZYXShuffle4Slice3 : IShuffle4Slice3 // Each four-byte group is an XYZW pixel. Selecting [3, 2, 1, 0] produces // WZYX, and offsets 4, 8, and 12 repeat that reversal for the next pixels. // The surrounding pipeline subsequently removes every fourth byte. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); } /// @@ -95,7 +95,7 @@ internal readonly struct ZYXWShuffle4Slice3 : IShuffle4Slice3 // Each four-byte group is an XYZW pixel. Selecting [2, 1, 0, 3] produces // ZYXW, and offsets 4, 8, and 12 repeat that exchange for the next pixels. // The surrounding pipeline subsequently removes every fourth byte. - => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); + => Vector128.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); } /// diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs b/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs index 503a64b90..1740df5b5 100644 --- a/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs +++ b/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs @@ -359,17 +359,17 @@ internal static partial class SimdUtils ref Vector512 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector512 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector512_.ShuffleNative(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector512_.ShuffleNative(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector512_.ShuffleNative(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector512_.ShuffleNative(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector512.ShuffleNative(vs0, mask); + Unsafe.Add(ref vd0, (nuint)1) = Vector512.ShuffleNative(Unsafe.Add(ref vs0, (nuint)1), mask); + Unsafe.Add(ref vd0, (nuint)2) = Vector512.ShuffleNative(Unsafe.Add(ref vs0, (nuint)2), mask); + Unsafe.Add(ref vd0, (nuint)3) = Vector512.ShuffleNative(Unsafe.Add(ref vs0, (nuint)3), mask); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector512_.ShuffleNative(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector512.ShuffleNative(Unsafe.Add(ref sourceBase, i), mask); } } } @@ -428,17 +428,17 @@ internal static partial class SimdUtils ref Vector128 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector128 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector128_.ShuffleNative(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector128_.ShuffleNative(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector128_.ShuffleNative(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector128_.ShuffleNative(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector128.ShuffleNative(vs0, mask); + Unsafe.Add(ref vd0, (nuint)1) = Vector128.ShuffleNative(Unsafe.Add(ref vs0, (nuint)1), mask); + Unsafe.Add(ref vd0, (nuint)2) = Vector128.ShuffleNative(Unsafe.Add(ref vs0, (nuint)2), mask); + Unsafe.Add(ref vd0, (nuint)3) = Vector128.ShuffleNative(Unsafe.Add(ref vs0, (nuint)3), mask); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector128_.ShuffleNative(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector128.ShuffleNative(Unsafe.Add(ref sourceBase, i), mask); } } } @@ -477,15 +477,15 @@ internal static partial class SimdUtils v2 = Vector128_.AlignRight(v2, v1, 8); v1 = Vector128_.AlignRight(v1, v0, 12); - v0 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v0, maskPad4Nx16), mask); - v1 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v1, maskPad4Nx16), mask); - v2 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v2, maskPad4Nx16), mask); - v3 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v3, maskPad4Nx16), mask); + v0 = Vector128.ShuffleNative(Vector128.ShuffleNative(v0, maskPad4Nx16), mask); + v1 = Vector128.ShuffleNative(Vector128.ShuffleNative(v1, maskPad4Nx16), mask); + v2 = Vector128.ShuffleNative(Vector128.ShuffleNative(v2, maskPad4Nx16), mask); + v3 = Vector128.ShuffleNative(Vector128.ShuffleNative(v3, maskPad4Nx16), mask); - v0 = Vector128_.ShuffleNative(v0, maskE); - v1 = Vector128_.ShuffleNative(v1, maskSlice4Nx16); - v2 = Vector128_.ShuffleNative(v2, maskE); - v3 = Vector128_.ShuffleNative(v3, maskSlice4Nx16); + v0 = Vector128.ShuffleNative(v0, maskE); + v1 = Vector128.ShuffleNative(v1, maskSlice4Nx16); + v2 = Vector128.ShuffleNative(v2, maskE); + v3 = Vector128.ShuffleNative(v3, maskSlice4Nx16); v0 = Vector128_.AlignRight(v1, v0, 4); v3 = Vector128_.AlignRight(v3, v2, 12); @@ -539,10 +539,10 @@ internal static partial class SimdUtils ref Vector128 vd = ref Unsafe.Add(ref destinationBase, j); - vd = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v0, maskPad4Nx16) | fill, mask); - Unsafe.Add(ref vd, 1) = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v1, maskPad4Nx16) | fill, mask); - Unsafe.Add(ref vd, 2) = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v2, maskPad4Nx16) | fill, mask); - Unsafe.Add(ref vd, 3) = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v3, maskPad4Nx16) | fill, mask); + vd = Vector128.ShuffleNative(Vector128.ShuffleNative(v0, maskPad4Nx16) | fill, mask); + Unsafe.Add(ref vd, 1) = Vector128.ShuffleNative(Vector128.ShuffleNative(v1, maskPad4Nx16) | fill, mask); + Unsafe.Add(ref vd, 2) = Vector128.ShuffleNative(Vector128.ShuffleNative(v2, maskPad4Nx16) | fill, mask); + Unsafe.Add(ref vd, 3) = Vector128.ShuffleNative(Vector128.ShuffleNative(v3, maskPad4Nx16) | fill, mask); } } } @@ -579,10 +579,10 @@ internal static partial class SimdUtils Vector128 v2 = Unsafe.Add(ref vs, 2); Vector128 v3 = Unsafe.Add(ref vs, 3); - v0 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v0, mask), maskE); - v1 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v1, mask), maskSlice4Nx16); - v2 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v2, mask), maskE); - v3 = Vector128_.ShuffleNative(Vector128_.ShuffleNative(v3, mask), maskSlice4Nx16); + v0 = Vector128.ShuffleNative(Vector128.ShuffleNative(v0, mask), maskE); + v1 = Vector128.ShuffleNative(Vector128.ShuffleNative(v1, mask), maskSlice4Nx16); + v2 = Vector128.ShuffleNative(Vector128.ShuffleNative(v2, mask), maskE); + v3 = Vector128.ShuffleNative(Vector128.ShuffleNative(v3, mask), maskSlice4Nx16); v0 = Vector128_.AlignRight(v1, v0, 4); v3 = Vector128_.AlignRight(v3, v2, 12); @@ -740,10 +740,10 @@ internal static partial class SimdUtils Vector512 f3 = Avx512F.ConvertToVector512Single(i3); // The residual term restores the correctly rounded byte / 255F result without paying for vector division. - f0 = Vector512_.FusedMultiplyAdd(f0, high, f0 * low); - f1 = Vector512_.FusedMultiplyAdd(f1, high, f1 * low); - f2 = Vector512_.FusedMultiplyAdd(f2, high, f2 * low); - f3 = Vector512_.FusedMultiplyAdd(f3, high, f3 * low); + f0 = Vector512.FusedMultiplyAdd(f0, high, f0 * low); + f1 = Vector512.FusedMultiplyAdd(f1, high, f1 * low); + f2 = Vector512.FusedMultiplyAdd(f2, high, f2 * low); + f3 = Vector512.FusedMultiplyAdd(f3, high, f3 * low); ref Vector512 d = ref Unsafe.Add(ref destinationBase, i * 4); @@ -780,10 +780,10 @@ internal static partial class SimdUtils Vector256 f2 = Avx.ConvertToVector256Single(i2); Vector256 f3 = Avx.ConvertToVector256Single(i3); - f0 = Vector256_.FusedMultiplyAdd(f0, high, f0 * low); - f1 = Vector256_.FusedMultiplyAdd(f1, high, f1 * low); - f2 = Vector256_.FusedMultiplyAdd(f2, high, f2 * low); - f3 = Vector256_.FusedMultiplyAdd(f3, high, f3 * low); + f0 = Vector256.FusedMultiplyAdd(f0, high, f0 * low); + f1 = Vector256.FusedMultiplyAdd(f1, high, f1 * low); + f2 = Vector256.FusedMultiplyAdd(f2, high, f2 * low); + f3 = Vector256.FusedMultiplyAdd(f3, high, f3 * low); ref Vector256 d = ref Unsafe.Add(ref destinationBase, i * 4); @@ -833,10 +833,10 @@ internal static partial class SimdUtils Vector128 f2 = Vector128.ConvertToSingle(i2); Vector128 f3 = Vector128.ConvertToSingle(i3); - f0 = Vector128_.FusedMultiplyAdd(f0, high, f0 * low); - f1 = Vector128_.FusedMultiplyAdd(f1, high, f1 * low); - f2 = Vector128_.FusedMultiplyAdd(f2, high, f2 * low); - f3 = Vector128_.FusedMultiplyAdd(f3, high, f3 * low); + f0 = Vector128.FusedMultiplyAdd(f0, high, f0 * low); + f1 = Vector128.FusedMultiplyAdd(f1, high, f1 * low); + f2 = Vector128.FusedMultiplyAdd(f2, high, f2 * low); + f3 = Vector128.FusedMultiplyAdd(f3, high, f3 * low); ref Vector128 d = ref Unsafe.Add(ref destinationBase, i * 4); @@ -1026,10 +1026,10 @@ internal static partial class SimdUtils Vector128 w2 = Vector128_.ConvertToInt32RoundAwayFromZero(f2); Vector128 w3 = Vector128_.ConvertToInt32RoundAwayFromZero(f3); - w0 = Vector128_.Clamp(w0, min, max); - w1 = Vector128_.Clamp(w1, min, max); - w2 = Vector128_.Clamp(w2, min, max); - w3 = Vector128_.Clamp(w3, min, max); + w0 = Vector128.Clamp(w0, min, max); + w1 = Vector128.Clamp(w1, min, max); + w2 = Vector128.Clamp(w2, min, max); + w3 = Vector128.Clamp(w3, min, max); Vector128 u0 = Vector128.Narrow(w0, w1).AsUInt16(); Vector128 u1 = Vector128.Narrow(w2, w3).AsUInt16(); diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs b/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs index 77f2de86c..afccdc63b 100644 --- a/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs +++ b/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs @@ -182,15 +182,15 @@ internal static partial class SimdUtils v2 = Vector128_.AlignRight(v2, v1, 8); v1 = Vector128_.AlignRight(v1, v0, 12); - v0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask)); - v1 = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask)); - v2 = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask)); - v3 = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask)); + v0 = TShuffle.Invoke(Vector128.ShuffleNative(v0, padMask)); + v1 = TShuffle.Invoke(Vector128.ShuffleNative(v1, padMask)); + v2 = TShuffle.Invoke(Vector128.ShuffleNative(v2, padMask)); + v3 = TShuffle.Invoke(Vector128.ShuffleNative(v3, padMask)); - v0 = Vector128_.ShuffleNative(v0, sliceEndMask); - v1 = Vector128_.ShuffleNative(v1, sliceMask); - v2 = Vector128_.ShuffleNative(v2, sliceEndMask); - v3 = Vector128_.ShuffleNative(v3, sliceMask); + v0 = Vector128.ShuffleNative(v0, sliceEndMask); + v1 = Vector128.ShuffleNative(v1, sliceMask); + v2 = Vector128.ShuffleNative(v2, sliceEndMask); + v3 = Vector128.ShuffleNative(v3, sliceMask); Vector128 destination0 = Vector128_.AlignRight(v1, v0, 4); Vector128 destination2 = Vector128_.AlignRight(v3, v2, 12); @@ -213,9 +213,9 @@ internal static partial class SimdUtils // the following pixels. The pad mask ignores those extra bytes before the operator // runs, and the slice mask packs the four results into the low twelve bytes. Vector128 result = Vector128.LoadUnsafe(ref sourceBase, (nuint)i); - result = Vector128_.ShuffleNative(result, padMask); + result = Vector128.ShuffleNative(result, padMask); result = TShuffle.Invoke(result); - result = Vector128_.ShuffleNative(result, sliceMask); + result = Vector128.ShuffleNative(result, sliceMask); // Store exactly twelve bytes so an in-place shuffle does not overwrite the next // source triplet captured by the following iteration. @@ -300,10 +300,10 @@ internal static partial class SimdUtils v1 = Vector128_.AlignRight(v1, v0, 12); ref Vector128 destination0 = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex); - destination0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask) | opaqueAlpha); - Unsafe.Add(ref destination0, 1) = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask) | opaqueAlpha); - Unsafe.Add(ref destination0, 2) = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask) | opaqueAlpha); - Unsafe.Add(ref destination0, 3) = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask) | opaqueAlpha); + destination0 = TShuffle.Invoke(Vector128.ShuffleNative(v0, padMask) | opaqueAlpha); + Unsafe.Add(ref destination0, 1) = TShuffle.Invoke(Vector128.ShuffleNative(v1, padMask) | opaqueAlpha); + Unsafe.Add(ref destination0, 2) = TShuffle.Invoke(Vector128.ShuffleNative(v2, padMask) | opaqueAlpha); + Unsafe.Add(ref destination0, 3) = TShuffle.Invoke(Vector128.ShuffleNative(v3, padMask) | opaqueAlpha); } sourceOffset = (int)(sourceVectorIndex * (uint)Vector128.Count); @@ -381,10 +381,10 @@ internal static partial class SimdUtils Vector128 v2 = TShuffle.Invoke(Unsafe.Add(ref source0, 2)); Vector128 v3 = TShuffle.Invoke(Unsafe.Add(ref source0, 3)); - v0 = Vector128_.ShuffleNative(v0, sliceEndMask); - v1 = Vector128_.ShuffleNative(v1, sliceMask); - v2 = Vector128_.ShuffleNative(v2, sliceEndMask); - v3 = Vector128_.ShuffleNative(v3, sliceMask); + v0 = Vector128.ShuffleNative(v0, sliceEndMask); + v1 = Vector128.ShuffleNative(v1, sliceMask); + v2 = Vector128.ShuffleNative(v2, sliceEndMask); + v3 = Vector128.ShuffleNative(v3, sliceMask); Vector128 destination0 = Vector128_.AlignRight(v1, v0, 4); Vector128 destination2 = Vector128_.AlignRight(v3, v2, 12); @@ -408,7 +408,7 @@ internal static partial class SimdUtils // One fixed shuffle then compacts four pixels into the low twelve vector bytes. Vector128 result = TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)sourceOffset)); - result = Vector128_.ShuffleNative(result, sliceMask); + result = Vector128.ShuffleNative(result, sliceMask); // The split store writes the exact 12-byte result and remains safe for in-place shrinking. Unsafe.As>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = result.GetLower(); diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs deleted file mode 100644 index 19f46c06f..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Add.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -internal static partial class TensorPrimitives_ -{ - /// - /// Computes the element-wise sum of the values in and . - /// - /// The element type. - /// The first addends. - /// The second addends. - /// The destination for the sums. - /// and do not have the same length. - /// is shorter than the input spans. - /// - /// An input and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IAdditionOperators, IAdditiveIdentity - => InvokeSpanSpanIntoSpan>(x, y, destination); - - /// - /// Computes the element-wise sum of the values in and the scalar . - /// - /// The element type. - /// The first addends. - /// The scalar second addend. - /// The destination for the sums. - /// is shorter than . - /// - /// and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ReadOnlySpan x, T y, Span destination) - where T : IAdditionOperators, IAdditiveIdentity - => InvokeSpanScalarIntoSpan>(x, y, destination); - - /// - /// Adds corresponding values. - /// - /// The element type. - private readonly struct AddOperator : IBinaryOperator - where T : IAdditionOperators, IAdditiveIdentity - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Adds scalar values. - /// - /// The first addend. - /// The second addend. - /// The sum. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => x + y; - - /// - /// Adds 128-bit vectors. - /// - /// The first addends. - /// The second addends. - /// The sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y; - - /// - /// Adds 256-bit vectors. - /// - /// The first addends. - /// The second addends. - /// The sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y; - - /// - /// Adds 512-bit vectors. - /// - /// The first addends. - /// The second addends. - /// The sums. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y; - } -} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs deleted file mode 100644 index b974163dc..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Clamp.cs +++ /dev/null @@ -1,322 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -internal static partial class TensorPrimitives_ -{ - /// - /// Computes the element-wise result of clamping to the inclusive range specified - /// by and . - /// - /// The element type. - /// The values to clamp. - /// The inclusive lower bound. - /// The inclusive upper bound. - /// The destination for the clamped values. - /// is shorter than . - /// - /// and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Clamp(ReadOnlySpan x, T min, T max, Span destination) - where T : INumber - => InvokeSpanScalarScalarIntoSpan>(x, min, max, destination); - - /// - /// Clamps single-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 ClampSingle( - Vector128 value, - Vector128 min, - Vector128 max) - { - // Unlike the native x86 min/max instructions, the normalized runtime operations propagate a NaN in the - // first operand and select negative zero when equal values have different signs. - Vector128 maximum = Vector128.ConditionalSelect( - Vector128.LessThan(min, value) - | ~Vector128.Equals(value, value) - | (Vector128.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), - value, - min); - - return Vector128.ConditionalSelect( - Vector128.LessThan(maximum, max) - | ~Vector128.Equals(maximum, maximum) - | (Vector128.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), - maximum, - max); - } - - /// - /// Clamps single-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 ClampSingle( - Vector256 value, - Vector256 min, - Vector256 max) - { - Vector256 maximum = Vector256.ConditionalSelect( - Vector256.LessThan(min, value) - | ~Vector256.Equals(value, value) - | (Vector256.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), - value, - min); - - return Vector256.ConditionalSelect( - Vector256.LessThan(maximum, max) - | ~Vector256.Equals(maximum, maximum) - | (Vector256.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), - maximum, - max); - } - - /// - /// Clamps single-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 ClampSingle( - Vector512 value, - Vector512 min, - Vector512 max) - { - Vector512 maximum = Vector512.ConditionalSelect( - Vector512.LessThan(min, value) - | ~Vector512.Equals(value, value) - | (Vector512.Equals(value, min) & (min.AsInt32() >> 31).AsSingle()), - value, - min); - - return Vector512.ConditionalSelect( - Vector512.LessThan(maximum, max) - | ~Vector512.Equals(maximum, maximum) - | (Vector512.Equals(maximum, max) & (maximum.AsInt32() >> 31).AsSingle()), - maximum, - max); - } - - /// - /// Clamps double-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 ClampDouble( - Vector128 value, - Vector128 min, - Vector128 max) - { - Vector128 maximum = Vector128.ConditionalSelect( - Vector128.LessThan(min, value) - | ~Vector128.Equals(value, value) - | (Vector128.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), - value, - min); - - return Vector128.ConditionalSelect( - Vector128.LessThan(maximum, max) - | ~Vector128.Equals(maximum, maximum) - | (Vector128.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), - maximum, - max); - } - - /// - /// Clamps double-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 ClampDouble( - Vector256 value, - Vector256 min, - Vector256 max) - { - Vector256 maximum = Vector256.ConditionalSelect( - Vector256.LessThan(min, value) - | ~Vector256.Equals(value, value) - | (Vector256.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), - value, - min); - - return Vector256.ConditionalSelect( - Vector256.LessThan(maximum, max) - | ~Vector256.Equals(maximum, maximum) - | (Vector256.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), - maximum, - max); - } - - /// - /// Clamps double-precision values with the normalized runtime semantics. - /// - /// The values to clamp. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 ClampDouble( - Vector512 value, - Vector512 min, - Vector512 max) - { - Vector512 maximum = Vector512.ConditionalSelect( - Vector512.LessThan(min, value) - | ~Vector512.Equals(value, value) - | (Vector512.Equals(value, min) & (min.AsInt64() >> 63).AsDouble()), - value, - min); - - return Vector512.ConditionalSelect( - Vector512.LessThan(maximum, max) - | ~Vector512.Equals(maximum, maximum) - | (Vector512.Equals(maximum, max) & (maximum.AsInt64() >> 63).AsDouble()), - maximum, - max); - } - - /// - /// Clamps values using the complete runtime tensor contract, including signed-zero correction. - /// - /// The element type. - private readonly struct ClampOperator : ITernaryOperator - where T : INumber - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Clamps a scalar value. - /// - /// The value. - /// The inclusive lower bound. - /// The inclusive upper bound. - /// The clamped value. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T min, T max) - => Vector128.IsSupported ? T.Min(T.Max(x, min), max) : T.Clamp(x, min, max); - - /// - /// Clamps a 128-bit vector. - /// - /// The values. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 min, Vector128 max) - { - if (typeof(T) == typeof(float)) - { - Vector128 result = ClampSingle( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref min), - Unsafe.As, Vector128>(ref max)); - - return Unsafe.As, Vector128>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector128 result = ClampDouble( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref min), - Unsafe.As, Vector128>(ref max)); - - return Unsafe.As, Vector128>(ref result); - } - - return Vector128_.Clamp(x, min, max); - } - - /// - /// Clamps a 256-bit vector. - /// - /// The values. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 min, Vector256 max) - { - if (typeof(T) == typeof(float)) - { - Vector256 result = ClampSingle( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref min), - Unsafe.As, Vector256>(ref max)); - - return Unsafe.As, Vector256>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector256 result = ClampDouble( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref min), - Unsafe.As, Vector256>(ref max)); - - return Unsafe.As, Vector256>(ref result); - } - - return Vector256_.Clamp(x, min, max); - } - - /// - /// Clamps a 512-bit vector. - /// - /// The values. - /// The inclusive lower bounds. - /// The inclusive upper bounds. - /// The clamped values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 min, Vector512 max) - { - if (typeof(T) == typeof(float)) - { - Vector512 result = ClampSingle( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref min), - Unsafe.As, Vector512>(ref max)); - - return Unsafe.As, Vector512>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector512 result = ClampDouble( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref min), - Unsafe.As, Vector512>(ref max)); - - return Unsafe.As, Vector512>(ref result); - } - - return Vector512_.Clamp(x, min, max); - } - } -} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs deleted file mode 100644 index f4a5bd01a..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Divide.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -internal static partial class TensorPrimitives_ -{ - /// - /// Computes the element-wise result of dividing the values in by . - /// - /// The element type. - /// The dividend values. - /// The divisor. - /// The destination for the quotient values. - /// is shorter than . - /// - /// and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Divide(ReadOnlySpan x, T y, Span destination) - where T : IDivisionOperators - => InvokeSpanScalarIntoSpanForDivision>(x, y, destination); - - /// - /// Determines whether has the same vector division support as . - /// - /// The element type. - /// when is a 32-bit signed native integer type. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsInt32Like() - => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); - - /// - /// Divides values by a scalar. - /// - /// The element type. - private readonly struct DivideOperator : IBinaryOperator - where T : IDivisionOperators - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => typeof(T) == typeof(float) - || typeof(T) == typeof(double) - || (Vector256.IsHardwareAccelerated && IsInt32Like()); - - /// - /// Divides scalar values. - /// - /// The dividend. - /// The divisor. - /// The quotient. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => x / y; - - /// - /// Divides 128-bit vectors. - /// - /// The dividends. - /// The divisors. - /// The quotients. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; - - /// - /// Divides 256-bit vectors. - /// - /// The dividends. - /// The divisors. - /// The quotients. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; - - /// - /// Divides 512-bit vectors. - /// - /// The dividends. - /// The divisors. - /// The quotients. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; - } -} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs deleted file mode 100644 index e80ca5411..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Helpers.cs +++ /dev/null @@ -1,900 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Diagnostics.CodeAnalysis; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -/// -/// Provides compatibility implementations for tensor operations that are not available on every target framework. -/// -/// -/// The API shape follows System.Numerics.Tensors.TensorPrimitives so call sites can move to the runtime -/// implementation when ImageSharp no longer supports target frameworks that predate it. -/// -internal static partial class TensorPrimitives_ -{ - /// - /// Defines an element-wise binary operation. - /// - /// The element type. - private interface IBinaryOperator - { - /// - /// Gets a value indicating whether the operation supports vector execution. - /// - public static abstract bool Vectorizable { get; } - - /// - /// Applies the operation to scalar values. - /// - /// The first value. - /// The second value. - /// The operation result. - public static abstract T Invoke(T x, T y); - - /// - /// Applies the operation to 128-bit vectors. - /// - /// The first vector. - /// The second vector. - /// The operation result. - public static abstract Vector128 Invoke(Vector128 x, Vector128 y); - - /// - /// Applies the operation to 256-bit vectors. - /// - /// The first vector. - /// The second vector. - /// The operation result. - public static abstract Vector256 Invoke(Vector256 x, Vector256 y); - - /// - /// Applies the operation to 512-bit vectors. - /// - /// The first vector. - /// The second vector. - /// The operation result. - public static abstract Vector512 Invoke(Vector512 x, Vector512 y); - } - - /// - /// Defines an element-wise ternary operation. - /// - /// The element type. - private interface ITernaryOperator - { - /// - /// Gets a value indicating whether the operation supports vector execution. - /// - public static abstract bool Vectorizable { get; } - - /// - /// Applies the operation to scalar values. - /// - /// The first value. - /// The second value. - /// The third value. - /// The operation result. - public static abstract T Invoke(T x, T y, T z); - - /// - /// Applies the operation to 128-bit vectors. - /// - /// The first vector. - /// The second vector. - /// The third vector. - /// The operation result. - public static abstract Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z); - - /// - /// Applies the operation to 256-bit vectors. - /// - /// The first vector. - /// The second vector. - /// The third vector. - /// The operation result. - public static abstract Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z); - - /// - /// Applies the operation to 512-bit vectors. - /// - /// The first vector. - /// The second vector. - /// The third vector. - /// The operation result. - public static abstract Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z); - } - - /// - /// Validates that an input and destination are either disjoint or begin at the same memory location. - /// - /// The element type. - /// The input values. - /// The destination values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void ValidateInputOutputSpanNonOverlapping(ReadOnlySpan input, Span destination) - { - // Runtime TensorPrimitives permits exact same-start overlap for in-place operation. A shifted overlap is - // rejected because forward SIMD stores could overwrite input elements before a later load consumes them. - if (!Unsafe.AreSame(ref MemoryMarshal.GetReference(input), ref MemoryMarshal.GetReference(destination)) - && input.Overlaps(destination)) - { - ThrowInputAndDestinationSpanMustNotOverlap(); - } - } - - /// - /// Throws when input spans do not have the same length. - /// - [DoesNotReturn] - private static void ThrowSpansMustHaveSameLength() - => throw new ArgumentException("Input span arguments must all have the same length."); - - /// - /// Throws when the destination cannot hold every result. - /// - [DoesNotReturn] - private static void ThrowDestinationTooShort() - => throw new ArgumentException("Destination is too short.", "destination"); - - /// - /// Throws when an input and destination overlap without beginning at the same memory location. - /// - [DoesNotReturn] - private static void ThrowInputAndDestinationSpanMustNotOverlap() - => throw new ArgumentException( - "The destination span may only overlap with an input span if the two spans start at the same memory location.", - "destination"); - - /// - /// Performs an element-wise binary operation between two spans. - /// - /// The element type. - /// The operation to apply. - /// The first input values. - /// The second input values. - /// The destination values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeSpanSpanIntoSpan( - ReadOnlySpan x, - ReadOnlySpan y, - Span destination) - where TOperator : struct, IBinaryOperator - { - if (x.Length != y.Length) - { - ThrowSpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowDestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - ValidateInputOutputSpanNonOverlapping(y, destination); - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - ref T destinationRef = ref MemoryMarshal.GetReference(destination); - nuint length = (uint)x.Length; - - // Runtime main selects the widest supported pipeline once one complete vector is available. - // Each pipeline preloads its final inputs when a tail overlaps so same-start in-place operation remains correct. - if (TOperator.Vectorizable - && Vector512.IsHardwareAccelerated - && Vector512.IsSupported - && length >= (uint)Vector512.Count) - { - InvokeVectorized512(ref xRef, ref yRef, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) - { - InvokeVectorized256(ref xRef, ref yRef, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) - { - InvokeVectorized128(ref xRef, ref yRef, ref destinationRef, length); - return; - } - - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), Unsafe.Add(ref yRef, i)); - } - } - - /// - /// Performs an element-wise binary operation between a span and a scalar. - /// - /// The element type. - /// The operation to apply. - /// The input values. - /// The scalar input. - /// The destination values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeSpanScalarIntoSpan( - ReadOnlySpan x, - T y, - Span destination) - where TOperator : struct, IBinaryOperator - { - if (x.Length > destination.Length) - { - ThrowDestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T destinationRef = ref MemoryMarshal.GetReference(destination); - nuint length = (uint)x.Length; - - // Runtime main selects the widest supported pipeline once one complete vector is available. - if (TOperator.Vectorizable - && Vector512.IsHardwareAccelerated - && Vector512.IsSupported - && length >= (uint)Vector512.Count) - { - InvokeVectorized512(ref xRef, y, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) - { - InvokeVectorized256(ref xRef, y, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) - { - InvokeVectorized128(ref xRef, y, ref destinationRef, length); - return; - } - - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); - } - } - - /// - /// Performs element-wise division using the runtime tensor width-selection order. - /// - /// The element type. - /// The division operation to apply. - /// The input values. - /// The scalar divisor. - /// The destination values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeSpanScalarIntoSpanForDivision( - ReadOnlySpan x, - T y, - Span destination) - where TOperator : struct, IBinaryOperator - { - if (x.Length > destination.Length) - { - ThrowDestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T destinationRef = ref MemoryMarshal.GetReference(destination); - nuint length = (uint)x.Length; - - // Runtime main selects the widest supported pipeline once one complete vector is available. - if (TOperator.Vectorizable - && Vector512.IsHardwareAccelerated - && Vector512.IsSupported - && length >= (uint)Vector512.Count) - { - InvokeVectorized512(ref xRef, y, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) - { - InvokeVectorized256(ref xRef, y, ref destinationRef, length); - return; - } - - // Four values fill one 128-bit float vector. Processing exactly one packed prefix before the scalar - // remainder avoids the overlapping second vector that regresses the common seven-element normalization. - if (TOperator.Vectorizable - && Vector128.IsHardwareAccelerated - && Vector128.IsSupported - && length >= (uint)Vector128.Count) - { - nuint vectorCount = (uint)Vector128.Count; - Vector128 yVector = Vector128.Create(y); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef), yVector).StoreUnsafe(ref destinationRef); - - for (nuint i = vectorCount; i < length; i++) - { - Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); - } - - return; - } - - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y); - } - } - - /// - /// Performs an element-wise ternary operation between a span and two scalars. - /// - /// The element type. - /// The operation to apply. - /// The input values. - /// The first scalar input. - /// The second scalar input. - /// The destination values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeSpanScalarScalarIntoSpan( - ReadOnlySpan x, - T y, - T z, - Span destination) - where TOperator : struct, ITernaryOperator - { - if (x.Length > destination.Length) - { - ThrowDestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T destinationRef = ref MemoryMarshal.GetReference(destination); - nuint length = (uint)x.Length; - - // This dispatch mirrors the runtime pipeline: large inputs use the widest available registers while - // short inputs fall through to a width that fits, keeping the operator contract identical at every length. - if (TOperator.Vectorizable && Vector512.IsHardwareAccelerated && Vector512.IsSupported && length >= (uint)Vector512.Count) - { - InvokeVectorized512(ref xRef, y, z, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) - { - InvokeVectorized256(ref xRef, y, z, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) - { - InvokeVectorized128(ref xRef, y, z, ref destinationRef, length); - return; - } - - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i), y, z); - } - } - - /// - /// Applies a binary operation between two spans with 128-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first element of the first input. - /// The first element of the second input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized128( - ref T xRef, - ref T yRef, - ref T destinationRef, - nuint length) - where TOperator : struct, IBinaryOperator - { - nuint vectorCount = (uint)Vector128.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - - // When a tail exists, both final inputs are loaded before any stores. This permits either source to also - // be the destination when the tail starts inside the range written by the preceding full vector. - Vector128 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector128.LoadUnsafe(ref xRef, length - vectorCount), - Vector128.LoadUnsafe(ref yRef, length - vectorCount)); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector128.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), Vector128.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a binary operation between two spans with 256-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first element of the first input. - /// The first element of the second input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized256( - ref T xRef, - ref T yRef, - ref T destinationRef, - nuint length) - where TOperator : struct, IBinaryOperator - { - nuint vectorCount = (uint)Vector256.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector256 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector256.LoadUnsafe(ref xRef, length - vectorCount), - Vector256.LoadUnsafe(ref yRef, length - vectorCount)); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector256.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), Vector256.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a binary operation between two spans with 512-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first element of the first input. - /// The first element of the second input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized512( - ref T xRef, - ref T yRef, - ref T destinationRef, - nuint length) - where TOperator : struct, IBinaryOperator - { - nuint vectorCount = (uint)Vector512.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector512 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector512.LoadUnsafe(ref xRef, length - vectorCount), - Vector512.LoadUnsafe(ref yRef, length - vectorCount)); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), Vector512.LoadUnsafe(ref yRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), Vector512.LoadUnsafe(ref yRef, index)).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a binary operation between a span and a scalar with 128-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The scalar input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized128( - ref T xRef, - T y, - ref T destinationRef, - nuint length) - where TOperator : struct, IBinaryOperator - { - nuint vectorCount = (uint)Vector128.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector128 yVector = Vector128.Create(y); - - // When a tail exists, preloading its final vector is required for in-place operation because it must - // observe the original values before an earlier overlapping store writes them. - Vector128 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector128.LoadUnsafe(ref xRef, length - vectorCount), - yVector); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a binary operation with 256-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The scalar input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized256( - ref T xRef, - T y, - ref T destinationRef, - nuint length) - where TOperator : struct, IBinaryOperator - { - nuint vectorCount = (uint)Vector256.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector256 yVector = Vector256.Create(y); - Vector256 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector256.LoadUnsafe(ref xRef, length - vectorCount), - yVector); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a binary operation with 512-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The scalar input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized512( - ref T xRef, - T y, - ref T destinationRef, - nuint length) - where TOperator : struct, IBinaryOperator - { - nuint vectorCount = (uint)Vector512.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector512 yVector = Vector512.Create(y); - Vector512 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector512.LoadUnsafe(ref xRef, length - vectorCount), - yVector); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a ternary operation with 128-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The first scalar input. - /// The second scalar input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized128( - ref T xRef, - T y, - T z, - ref T destinationRef, - nuint length) - where TOperator : struct, ITernaryOperator - { - nuint vectorCount = (uint)Vector128.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector128 yVector = Vector128.Create(y); - Vector128 zVector = Vector128.Create(z); - Vector128 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector128.LoadUnsafe(ref xRef, length - vectorCount), - yVector, - zVector); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a ternary operation with 256-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The first scalar input. - /// The second scalar input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized256( - ref T xRef, - T y, - T z, - ref T destinationRef, - nuint length) - where TOperator : struct, ITernaryOperator - { - nuint vectorCount = (uint)Vector256.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector256 yVector = Vector256.Create(y); - Vector256 zVector = Vector256.Create(z); - Vector256 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector256.LoadUnsafe(ref xRef, length - vectorCount), - yVector, - zVector); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a ternary operation with 512-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The first scalar input. - /// The second scalar input. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeVectorized512( - ref T xRef, - T y, - T z, - ref T destinationRef, - nuint length) - where TOperator : struct, ITernaryOperator - { - nuint vectorCount = (uint)Vector512.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector512 yVector = Vector512.Create(y); - Vector512 zVector = Vector512.Create(z); - Vector512 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke( - Vector512.LoadUnsafe(ref xRef, length - vectorCount), - yVector, - zVector); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7)), yVector, zVector).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index), yVector, zVector).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } -} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs deleted file mode 100644 index fc621a122..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Max.cs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -internal static partial class TensorPrimitives_ -{ - /// - /// Computes the element-wise maximum of the values in and . - /// - /// The element type. - /// The values to compare. - /// The value to compare with each element. - /// The destination for the maximum values. - /// is shorter than . - /// - /// and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Max(ReadOnlySpan x, T y, Span destination) - where T : INumber - => InvokeSpanScalarIntoSpan>(x, y, destination); - - /// - /// Selects maximum single-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 MaxSingle(Vector128 x, Vector128 y) - { - // The .NET 8 operation already handles ordered unequal values. Correct its second-operand result for a - // first-operand NaN, then use bitwise AND for equal values so positive zero wins regardless of operand order. - Vector128 result = Vector128.Max(x, y); - result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); - - return Vector128.ConditionalSelect( - Vector128.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum single-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 MaxSingle(Vector256 x, Vector256 y) - { - Vector256 result = Vector256.Max(x, y); - result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); - - return Vector256.ConditionalSelect( - Vector256.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum single-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 MaxSingle(Vector512 x, Vector512 y) - { - Vector512 result = Vector512.Max(x, y); - result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); - - return Vector512.ConditionalSelect( - Vector512.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum double-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 MaxDouble(Vector128 x, Vector128 y) - { - Vector128 result = Vector128.Max(x, y); - result = Vector128.ConditionalSelect(~Vector128.Equals(x, x), x, result); - - return Vector128.ConditionalSelect( - Vector128.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum double-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 MaxDouble(Vector256 x, Vector256 y) - { - Vector256 result = Vector256.Max(x, y); - result = Vector256.ConditionalSelect(~Vector256.Equals(x, x), x, result); - - return Vector256.ConditionalSelect( - Vector256.Equals(x, y), - x & y, - result); - } - - /// - /// Selects maximum double-precision values with the normalized runtime semantics. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 MaxDouble(Vector512 x, Vector512 y) - { - Vector512 result = Vector512.Max(x, y); - result = Vector512.ConditionalSelect(~Vector512.Equals(x, x), x, result); - - return Vector512.ConditionalSelect( - Vector512.Equals(x, y), - x & y, - result); - } - - /// - /// Selects the maximum corresponding values. - /// - /// The element type. - private readonly struct MaxOperator : IBinaryOperator - where T : INumber - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Selects the maximum scalar value. - /// - /// The first value. - /// The second value. - /// The maximum value. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.Max(x, y); - - /// - /// Selects the maximum values from 128-bit vectors. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (typeof(T) == typeof(float)) - { - Vector128 result = MaxSingle( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref y)); - - return Unsafe.As, Vector128>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector128 result = MaxDouble( - Unsafe.As, Vector128>(ref x), - Unsafe.As, Vector128>(ref y)); - - return Unsafe.As, Vector128>(ref result); - } - - return Vector128.Max(x, y); - } - - /// - /// Selects the maximum values from 256-bit vectors. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float)) - { - Vector256 result = MaxSingle( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref y)); - - return Unsafe.As, Vector256>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector256 result = MaxDouble( - Unsafe.As, Vector256>(ref x), - Unsafe.As, Vector256>(ref y)); - - return Unsafe.As, Vector256>(ref result); - } - - return Vector256.Max(x, y); - } - - /// - /// Selects the maximum values from 512-bit vectors. - /// - /// The first values. - /// The second values. - /// The maximum values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float)) - { - Vector512 result = MaxSingle( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref y)); - - return Unsafe.As, Vector512>(ref result); - } - - if (typeof(T) == typeof(double)) - { - Vector512 result = MaxDouble( - Unsafe.As, Vector512>(ref x), - Unsafe.As, Vector512>(ref y)); - - return Unsafe.As, Vector512>(ref result); - } - - return Vector512.Max(x, y); - } - } -} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs deleted file mode 100644 index 367a7ab0b..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Multiply.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -internal static partial class TensorPrimitives_ -{ - /// - /// Computes the element-wise product of the values in and . - /// - /// The element type. - /// The multiplicands. - /// The multiplier. - /// The destination for the products. - /// is shorter than . - /// - /// and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Multiply(ReadOnlySpan x, T y, Span destination) - where T : IMultiplyOperators, IMultiplicativeIdentity - => InvokeSpanScalarIntoSpan>(x, y, destination); - - /// - /// Multiplies corresponding values. - /// - /// The element type. - private readonly struct MultiplyOperator : IBinaryOperator - where T : IMultiplyOperators, IMultiplicativeIdentity - { - /// - /// Gets a value indicating whether this operation supports vector execution. - /// - public static bool Vectorizable => true; - - /// - /// Multiplies scalar values. - /// - /// The multiplicand. - /// The multiplier. - /// The product. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => x * y; - - /// - /// Multiplies 128-bit vectors. - /// - /// The multiplicands. - /// The multipliers. - /// The products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) => x * y; - - /// - /// Multiplies 256-bit vectors. - /// - /// The multiplicands. - /// The multipliers. - /// The products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) => x * y; - - /// - /// Multiplies 512-bit vectors. - /// - /// The multiplicands. - /// The multipliers. - /// The products. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; - } -} diff --git a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs b/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs deleted file mode 100644 index 05f93ac66..000000000 --- a/src/ImageSharp/Common/Helpers/TensorPrimitives_.Negate.cs +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Common.Helpers; - -internal static partial class TensorPrimitives_ -{ - /// - /// Defines an element-wise unary operation. - /// - /// The element type. - private interface IUnaryOperator - { - /// - /// Gets a value indicating whether the operation supports vector execution. - /// - public static abstract bool Vectorizable { get; } - - /// - /// Applies the operation to a scalar value. - /// - /// The input value. - /// The operation result. - public static abstract T Invoke(T x); - - /// - /// Applies the operation to a 128-bit vector. - /// - /// The input vector. - /// The operation result. - public static abstract Vector128 Invoke(Vector128 x); - - /// - /// Applies the operation to a 256-bit vector. - /// - /// The input vector. - /// The operation result. - public static abstract Vector256 Invoke(Vector256 x); - - /// - /// Applies the operation to a 512-bit vector. - /// - /// The input vector. - /// The operation result. - public static abstract Vector512 Invoke(Vector512 x); - } - - /// - /// Computes the element-wise negation of the values in . - /// - /// The element type. - /// The values to negate. - /// The destination for the negated values. - /// is shorter than . - /// - /// and overlap without beginning at the same memory location. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Negate(ReadOnlySpan x, Span destination) - where T : IUnaryNegationOperators - => InvokeSpanIntoSpan>(x, destination); - - /// - /// Performs an element-wise unary operation over a span. - /// - /// The element type. - /// The operation to apply. - /// The input values. - /// The destination values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeSpanIntoSpan(ReadOnlySpan x, Span destination) - where TOperator : struct, IUnaryOperator - { - if (x.Length > destination.Length) - { - ThrowDestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T destinationRef = ref MemoryMarshal.GetReference(destination); - nuint length = (uint)x.Length; - - // Runtime main selects the widest supported pipeline once one complete vector is available. - if (TOperator.Vectorizable - && Vector512.IsHardwareAccelerated - && Vector512.IsSupported - && length >= (uint)Vector512.Count) - { - InvokeUnaryVectorized512(ref xRef, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256.IsSupported && length >= (uint)Vector256.Count) - { - InvokeUnaryVectorized256(ref xRef, ref destinationRef, length); - return; - } - - if (TOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128.IsSupported && length >= (uint)Vector128.Count) - { - InvokeUnaryVectorized128(ref xRef, ref destinationRef, length); - return; - } - - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref destinationRef, i) = TOperator.Invoke(Unsafe.Add(ref xRef, i)); - } - } - - /// - /// Applies a unary operation with 128-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeUnaryVectorized128(ref T xRef, ref T destinationRef, nuint length) - where TOperator : struct, IUnaryOperator - { - nuint vectorCount = (uint)Vector128.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - - // The final vector overlaps the preceding store when the length is not a vector multiple. Loading it - // before any stores preserves same-start in-place operation because it captures the original tail. - Vector128 end = default; - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, length - vectorCount)); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a unary operation with 256-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeUnaryVectorized256(ref T xRef, ref T destinationRef, nuint length) - where TOperator : struct, IUnaryOperator - { - nuint vectorCount = (uint)Vector256.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector256 end = default; - - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, length - vectorCount)); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Applies a unary operation with 512-bit vectors. - /// - /// The element type. - /// The operation to apply. - /// The first input element. - /// The first destination element. - /// The number of elements to process. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void InvokeUnaryVectorized512(ref T xRef, ref T destinationRef, nuint length) - where TOperator : struct, IUnaryOperator - { - nuint vectorCount = (uint)Vector512.Count; - nuint vectorsPerLoop = vectorCount * 8; - nuint index = 0; - Vector512 end = default; - - if ((length % vectorCount) != 0) - { - end = TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, length - vectorCount)); - } - - while ((length - index) >= vectorsPerLoop) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 0))).StoreUnsafe(ref destinationRef, index + (vectorCount * 0)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 1))).StoreUnsafe(ref destinationRef, index + (vectorCount * 1)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 2))).StoreUnsafe(ref destinationRef, index + (vectorCount * 2)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 3))).StoreUnsafe(ref destinationRef, index + (vectorCount * 3)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 4))).StoreUnsafe(ref destinationRef, index + (vectorCount * 4)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 5))).StoreUnsafe(ref destinationRef, index + (vectorCount * 5)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 6))).StoreUnsafe(ref destinationRef, index + (vectorCount * 6)); - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index + (vectorCount * 7))).StoreUnsafe(ref destinationRef, index + (vectorCount * 7)); - - index += vectorsPerLoop; - } - - while ((length - index) >= vectorCount) - { - TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, index)).StoreUnsafe(ref destinationRef, index); - index += vectorCount; - } - - if (index != length) - { - end.StoreUnsafe(ref destinationRef, length - vectorCount); - } - } - - /// - /// Implements element-wise negation for scalar and SIMD inputs. - /// - /// The element type. - private readonly struct NegateOperator : IUnaryOperator - where T : IUnaryNegationOperators - { - /// - public static bool Vectorizable => true; - - /// - public static T Invoke(T x) => -x; - - /// - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - // IEEE-754 negation toggles the sign bit. Expressing that operation explicitly avoids the - // subtraction-based ARM64 code generated by .NET 8 for generic vector negation, which loses - // the sign when +0F is negated and therefore differs from both scalar and runtime-main behavior. - return x ^ Vector128.Create(-0F).As(); - } - - if (typeof(T) == typeof(double)) - { - // Double-precision values use the same sign-bit representation, with the sign in bit 63. - return x ^ Vector128.Create(-0D).As(); - } - - return -x; - } - - /// - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - // Keep the operation bitwise at every width so ARM64 preserves signed zero exactly. - return x ^ Vector256.Create(-0F).As(); - } - - if (typeof(T) == typeof(double)) - { - return x ^ Vector256.Create(-0D).As(); - } - - return -x; - } - - /// - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - // Vector512 can be hardware accelerated directly or decomposed by the runtime; the explicit - // bit operation provides identical IEEE-754 behavior in either case. - return x ^ Vector512.Create(-0F).As(); - } - - if (typeof(T) == typeof(double)) - { - return x ^ Vector512.Create(-0D).As(); - } - - return -x; - } - } -} diff --git a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs index 4a53fdda4..6bb1f59ef 100644 --- a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs +++ b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs @@ -74,7 +74,7 @@ internal static class Vector128_ (control >> 4) & 0x3, (control >> 6) & 0x3); - return Vector128.Shuffle(vector, indices); + return Vector128.ShuffleNative(vector, indices); } /// @@ -93,7 +93,7 @@ internal static class Vector128_ (control >> 4) & 0x3, (control >> 6) & 0x3); - return Vector128.Shuffle(vector, indices); + return Vector128.ShuffleNative(vector, indices); } /// @@ -124,7 +124,7 @@ internal static class Vector128_ (short)(((control >> 4) & 0x3) + 4), (short)(((control >> 6) & 0x3) + 4)); - return Vector128.Shuffle(value, indices); + return Vector128.ShuffleNative(value, indices); } /// @@ -155,33 +155,7 @@ internal static class Vector128_ 6, 7); - return Vector128.Shuffle(value, indices); - } - - /// - /// Creates a new vector by selecting values from an input vector using a set of indices. - /// - /// - /// The input vector from which values are selected. - /// - /// The per-element indices used to select a value from . - /// - /// - /// A new vector containing the values from selected by the given . - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShuffleNative(Vector128 vector, Vector128 indices) - { - // For x64 we use the SSSE3 shuffle intrinsic to avoid additional instructions. 3 vs 1. - if (Ssse3.IsSupported) - { - return Ssse3.Shuffle(vector, indices); - } - - // For ARM and WASM, codegen will be optimal. - // We don't throw for x86/x64 so we should never use this method without - // checking for support. - return Vector128.Shuffle(vector, indices); + return Vector128.ShuffleNative(value, indices); } /// @@ -230,27 +204,6 @@ internal static class Vector128_ return Vector128.Shuffle(value, Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) - Vector128.Create(numBytes)); } - /// - /// Shift packed 16-bit integers in left by while - /// shifting in zeros, and store the results - /// - /// The vector containing packed 16-bit integers to shift. - /// The number of bits to shift left. - /// - /// A vector containing the packed 16-bit integers shifted left by , with zeros shifted in. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeftLogical(Vector128 value, [ConstantExpected] byte count) - { - // Zero lanes where count >= 16 to match SSE2 - if (count >= 16) - { - return Vector128.Zero; - } - - return value << count; - } - /// /// Right aligns elements of two source 128-bit values depending on bits in a mask. /// @@ -332,92 +285,6 @@ internal static class Vector128_ return Vector128.ConvertToInt32(vector + fallbackAdjustment); } - /// - /// Rounds all values in to the nearest integer - /// following semantics. - /// - /// The vector. - /// The vector with each value rounded to the nearest integer. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 RoundToNearestInteger(Vector128 vector) - { - if (Sse41.IsSupported) - { - return Sse41.RoundToNearestInteger(vector); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.RoundToNearest(vector); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.RoundToNearest(vector); - } - - Vector128 sign = vector & Vector128.Create(-0F); - Vector128 val_2p23_f32 = sign | Vector128.Create(8388608F); - - val_2p23_f32 = (vector + val_2p23_f32) - val_2p23_f32; - return val_2p23_f32 | sign; - } - - /// - /// Computes an estimate of ( * ) + . - /// - /// The first vector to multiply. - /// The second vector to multiply. - /// The vector to add to the product. - /// An estimate of the multiplication and addition result. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 MultiplyAddEstimate(Vector128 left, Vector128 right, Vector128 addend) - { - if (Fma.IsSupported) - { - return Fma.MultiplyAdd(left, right, addend); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.FusedMultiplyAdd(addend, left, right); - } - - return (left * right) + addend; - } - - /// - /// Computes ( * ) + , rounded as one ternary operation. - /// - /// The first vector to multiply. - /// The second vector to multiply. - /// The vector to add to the product. - /// The fused multiplication and addition result. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 FusedMultiplyAdd(Vector128 left, Vector128 right, Vector128 addend) - { - if (Fma.IsSupported) - { - return Fma.MultiplyAdd(left, right, addend); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.FusedMultiplyAdd(addend, left, right); - } - - // WebAssembly SIMD has no exact fused multiply-add, so match the runtime fallback by preserving fused rounding per element. - Vector64 lower = Vector64.Create( - MathF.FusedMultiplyAdd(left.GetElement(0), right.GetElement(0), addend.GetElement(0)), - MathF.FusedMultiplyAdd(left.GetElement(1), right.GetElement(1), addend.GetElement(1))); - - Vector64 upper = Vector64.Create( - MathF.FusedMultiplyAdd(left.GetElement(2), right.GetElement(2), addend.GetElement(2)), - MathF.FusedMultiplyAdd(left.GetElement(3), right.GetElement(3), addend.GetElement(3))); - - return Vector128.Create(lower, upper); - } - /// /// Packs signed 16-bit integers to unsigned 8-bit integers and saturates. /// @@ -443,8 +310,8 @@ internal static class Vector128_ Vector128 min = Vector128.Create((short)byte.MinValue); Vector128 max = Vector128.Create((short)byte.MaxValue); - Vector128 lefClamped = Clamp(left, min, max).AsUInt16(); - Vector128 rightClamped = Clamp(right, min, max).AsUInt16(); + Vector128 lefClamped = Vector128.Clamp(left, min, max).AsUInt16(); + Vector128 rightClamped = Vector128.Clamp(right, min, max).AsUInt16(); return Vector128.Narrow(lefClamped, rightClamped); } @@ -474,85 +341,11 @@ internal static class Vector128_ Vector128 min = Vector128.Create((int)ushort.MinValue); Vector128 max = Vector128.Create((int)ushort.MaxValue); - Vector128 lefClamped = Clamp(left, min, max).AsUInt32(); - Vector128 rightClamped = Clamp(right, min, max).AsUInt32(); + Vector128 lefClamped = Vector128.Clamp(left, min, max).AsUInt32(); + Vector128 rightClamped = Vector128.Clamp(right, min, max).AsUInt32(); return Vector128.Narrow(lefClamped, rightClamped); } - /// - /// Packs signed 32-bit integers to signed 16-bit integers and saturates. - /// - /// The left hand source vector. - /// The right hand source vector. - /// The . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 PackSignedSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.PackSignedSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(left), right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.ConvertNarrowingSaturateSigned(left, right); - } - - Vector128 min = Vector128.Create((int)short.MinValue); - Vector128 max = Vector128.Create((int)short.MaxValue); - Vector128 lefClamped = Clamp(left, min, max); - Vector128 rightClamped = Clamp(right, min, max); - return Vector128.Narrow(lefClamped, rightClamped); - } - - /// - /// Packs signed 16-bit integers to signed 8-bit integers and saturates. - /// - /// The left hand source vector. - /// The right hand source vector. - /// The . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 PackSignedSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.PackSignedSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(left), right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.ConvertNarrowingSaturateSigned(left, right); - } - - Vector128 min = Vector128.Create((short)sbyte.MinValue); - Vector128 max = Vector128.Create((short)sbyte.MaxValue); - Vector128 lefClamped = Clamp(left, min, max); - Vector128 rightClamped = Clamp(right, min, max); - return Vector128.Narrow(lefClamped, rightClamped); - } - - /// - /// Restricts a vector between a minimum and a maximum value. - /// - /// The type of the elements in the vector. - /// The vector to restrict. - /// The minimum value. - /// The maximum value. - /// The restricted . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Clamp(Vector128 value, Vector128 min, Vector128 max) - => Vector128.Min(Vector128.Max(value, min), max); - /// /// Multiply packed signed 16-bit integers in and , producing /// intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and @@ -847,7 +640,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetUpper(), right.GetUpper()); - return Vector128.Shuffle(unpacked, Vector128.Create(0, 2, 1, 3)); + return Vector128.ShuffleNative(unpacked, Vector128.Create(0, 2, 1, 3)); } /// @@ -878,7 +671,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetLower(), right.GetLower()); - return Vector128.Shuffle(unpacked, Vector128.Create(0, 2, 1, 3)); + return Vector128.ShuffleNative(unpacked, Vector128.Create(0, 2, 1, 3)); } /// @@ -909,7 +702,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetUpper(), right.GetUpper()); - return Vector128.Shuffle(unpacked, Vector128.Create(0, 4, 1, 5, 2, 6, 3, 7)); + return Vector128.ShuffleNative(unpacked, Vector128.Create(0, 4, 1, 5, 2, 6, 3, 7)); } /// @@ -940,7 +733,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetLower(), right.GetLower()); - return Vector128.Shuffle(unpacked, Vector128.Create(0, 4, 1, 5, 2, 6, 3, 7)); + return Vector128.ShuffleNative(unpacked, Vector128.Create(0, 4, 1, 5, 2, 6, 3, 7)); } /// @@ -971,7 +764,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetUpper(), right.GetUpper()); - return Vector128.Shuffle(unpacked, Vector128.Create((byte)0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); + return Vector128.ShuffleNative(unpacked, Vector128.Create((byte)0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); } /// @@ -1002,7 +795,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetLower(), right.GetLower()); - return Vector128.Shuffle(unpacked, Vector128.Create((byte)0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); + return Vector128.ShuffleNative(unpacked, Vector128.Create((byte)0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); } /// @@ -1033,7 +826,7 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetUpper(), right.GetUpper()); - return Vector128.Shuffle(unpacked, Vector128.Create(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); + return Vector128.ShuffleNative(unpacked, Vector128.Create(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); } /// @@ -1064,340 +857,6 @@ internal static class Vector128_ } Vector128 unpacked = Vector128.Create(left.GetLower(), right.GetLower()); - return Vector128.Shuffle(unpacked, Vector128.Create(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); - } - - /// - /// Subtract packed signed 16-bit integers in from packed signed 16-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed signed 16-bit integers to subtract from. - /// - /// - /// The second vector containing packed signed 16-bit integers to subtract. - /// - /// - /// A vector containing the results of subtracting packed signed 16-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 SubtractSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.SubtractSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.SubtractSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.SubtractSaturate(left, right); - } - - // Widen inputs to 32-bit signed - (Vector128 leftLo, Vector128 leftHi) = Vector128.Widen(left); - (Vector128 rightLo, Vector128 rightHi) = Vector128.Widen(right); - - // Subtract - Vector128 diffLo = leftLo - rightLo; - Vector128 diffHi = leftHi - rightHi; - - // Clamp to signed 16-bit range - Vector128 min = Vector128.Create((int)short.MinValue); - Vector128 max = Vector128.Create((int)short.MaxValue); - - diffLo = Clamp(diffLo, min, max); - diffHi = Clamp(diffHi, min, max); - - // Narrow back to 16 bit signed. - return Vector128.Narrow(diffLo, diffHi); - } - - /// - /// Subtract packed unsigned 16-bit integers in from packed unsigned 16-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed unsigned 16-bit integers to subtract from. - /// - /// - /// The second vector containing packed unsigned 16-bit integers to subtract. - /// - /// - /// A vector containing the results of subtracting packed unsigned 16-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 SubtractSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.SubtractSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.SubtractSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.SubtractSaturate(left, right); - } - - // Widen inputs to 32-bit signed - (Vector128 leftLo, Vector128 leftHi) = Vector128.Widen(left); - (Vector128 rightLo, Vector128 rightHi) = Vector128.Widen(right); - - // Subtract - Vector128 diffLo = leftLo - rightLo; - Vector128 diffHi = leftHi - rightHi; - - // Clamp to signed 16-bit range - Vector128 min = Vector128.Create((uint)ushort.MinValue); - Vector128 max = Vector128.Create((uint)ushort.MaxValue); - - diffLo = Clamp(diffLo, min, max); - diffHi = Clamp(diffHi, min, max); - - // Narrow back to 16 bit signed. - return Vector128.Narrow(diffLo, diffHi); - } - - /// - /// Add packed unsigned 8-bit integers in to packed unsigned 8-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed unsigned 8-bit integers to add to. - /// - /// - /// The second vector containing packed unsigned 8-bit integers to add. - /// - /// - /// A vector containing the results of adding packed unsigned 8-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 AddSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.AddSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.AddSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.AddSaturate(left, right); - } - - // Widen inputs to 16-bit - (Vector128 leftLo, Vector128 leftHi) = Vector128.Widen(left); - (Vector128 rightLo, Vector128 rightHi) = Vector128.Widen(right); - - // Add - Vector128 sumLo = leftLo + rightLo; - Vector128 sumHi = leftHi + rightHi; - - // Clamp to signed 8-bit range - Vector128 max = Vector128.Create((ushort)byte.MaxValue); - - sumLo = Clamp(sumLo, Vector128.Zero, max); - sumHi = Clamp(sumHi, Vector128.Zero, max); - - // Narrow back to bytes - return Vector128.Narrow(sumLo, sumHi); - } - - /// - /// Add packed unsigned 16-bit integers in to packed unsigned 16-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed unsigned 16-bit integers to add to. - /// - /// - /// The second vector containing packed unsigned 16-bit integers to add. - /// - /// - /// A vector containing the results of adding packed unsigned 16-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 AddSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.AddSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.AddSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.AddSaturate(left, right); - } - - // Widen inputs to 32-bit - (Vector128 leftLo, Vector128 leftHi) = Vector128.Widen(left); - (Vector128 rightLo, Vector128 rightHi) = Vector128.Widen(right); - - // Add - Vector128 sumLo = leftLo + rightLo; - Vector128 sumHi = leftHi + rightHi; - - // Clamp to signed 16-bit range - Vector128 max = Vector128.Create((uint)ushort.MaxValue); - - sumLo = Clamp(sumLo, Vector128.Zero, max); - sumHi = Clamp(sumHi, Vector128.Zero, max); - - // Narrow back to 16 bit unsigned. - return Vector128.Narrow(sumLo, sumHi); - } - - /// - /// Subtract packed unsigned 8-bit integers in from packed unsigned 8-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed unsigned 8-bit integers to subtract from. - /// - /// - /// The second vector containing packed unsigned 8-bit integers to subtract. - /// - /// - /// A vector containing the results of subtracting packed unsigned 8-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 SubtractSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.SubtractSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.SubtractSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.SubtractSaturate(left, right); - } - - // Subtracting the smaller operand implements the .NET 10 unsigned contract: - // lanes where right exceeds left subtract left from itself and therefore saturate at zero. - return left - Vector128.Min(left, right); - } - - /// - /// Add packed unsigned 8-bit integers in from packed unsigned 8-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed unsigned 8-bit integers to add to. - /// - /// - /// The second vector containing packed unsigned 8-bit integers to add. - /// - /// - /// A vector containing the results of adding packed unsigned 8-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 AddSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.AddSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.AddSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.AddSaturate(left, right); - } - - // Widen inputs to 16-bit - (Vector128 leftLo, Vector128 leftHi) = Vector128.Widen(left); - (Vector128 rightLo, Vector128 rightHi) = Vector128.Widen(right); - - // Add - Vector128 sumLo = leftLo + rightLo; - Vector128 sumHi = leftHi + rightHi; - - // Clamp to signed 8-bit range - Vector128 min = Vector128.Create((short)sbyte.MinValue); - Vector128 max = Vector128.Create((short)sbyte.MaxValue); - - sumLo = Clamp(sumLo, min, max); - sumHi = Clamp(sumHi, min, max); - - // Narrow back to signed bytes - return Vector128.Narrow(sumLo, sumHi); - } - - /// - /// Subtract packed signed 8-bit integers in from packed signed 8-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed signed 8-bit integers to subtract from. - /// - /// - /// The second vector containing packed signed 8-bit integers to subtract. - /// - /// - /// A vector containing the results of subtracting packed signed 8-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 SubtractSaturate(Vector128 left, Vector128 right) - { - if (Sse2.IsSupported) - { - return Sse2.SubtractSaturate(left, right); - } - - if (AdvSimd.IsSupported) - { - return AdvSimd.SubtractSaturate(left, right); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.SubtractSaturate(left, right); - } - - // Widen inputs to 16-bit - (Vector128 leftLo, Vector128 leftHi) = Vector128.Widen(left); - (Vector128 rightLo, Vector128 rightHi) = Vector128.Widen(right); - - // Subtract - Vector128 diffLo = leftLo - rightLo; - Vector128 diffHi = leftHi - rightHi; - - // Clamp to signed 8-bit range - Vector128 min = Vector128.Create((short)sbyte.MinValue); - Vector128 max = Vector128.Create((short)sbyte.MaxValue); - - diffLo = Clamp(diffLo, min, max); - diffHi = Clamp(diffHi, min, max); - - // Narrow back to signed bytes - return Vector128.Narrow(diffLo, diffHi); + return Vector128.ShuffleNative(unpacked, Vector128.Create(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)); } } diff --git a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs index 681f80013..1dd712271 100644 --- a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs +++ b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs @@ -31,13 +31,11 @@ internal static class Vector256_ => Avx.Shuffle(vector, vector, control); /// - /// Creates a new vector by selecting values from an input vector using a set of indices. - /// - /// The input vector from which values are selected. - /// - /// The per-element indices used to select a value from . - /// - /// The . + /// Creates a new vector by selecting values from each 128-bit input lane using the corresponding indices. + /// + /// The input vector from which values are selected. + /// The per-element indices used to select values within each 128-bit lane. + /// The shuffled . [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector256 ShufflePerLane(Vector256 vector, Vector256 indices) { @@ -46,9 +44,11 @@ internal static class Vector256_ return Avx2.Shuffle(vector, indices); } + // The .NET 10 fallback treats indices as full-width when AVX2 is unavailable. Reusing + // the low mask for each half preserves the lane-local vpshufb contract on AVX-only CPUs. Vector128 indicesLo = indices.GetLower(); - Vector128 lower = Vector128_.ShuffleNative(vector.GetLower(), indicesLo); - Vector128 upper = Vector128_.ShuffleNative(vector.GetUpper(), indicesLo); + Vector128 lower = Vector128.ShuffleNative(vector.GetLower(), indicesLo); + Vector128 upper = Vector128.ShuffleNative(vector.GetUpper(), indicesLo); return Vector256.Create(lower, upper); } @@ -93,70 +93,6 @@ internal static class Vector256_ return Vector256.ConvertToInt32(vector + fallbackAdjustment); } - /// - /// Rounds all values in to the nearest integer - /// following semantics. - /// - /// The vector. - /// The vector with each value rounded to the nearest integer. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 RoundToNearestInteger(Vector256 vector) - { - if (Avx.IsSupported) - { - return Avx.RoundToNearestInteger(vector); - } - - Vector256 sign = vector & Vector256.Create(-0F); - Vector256 val_2p23_f32 = sign | Vector256.Create(8388608F); - - val_2p23_f32 = (vector + val_2p23_f32) - val_2p23_f32; - return val_2p23_f32 | sign; - } - - /// - /// Computes an estimate of ( * ) + . - /// - /// The first vector to multiply. - /// The second vector to multiply. - /// The vector to add to the product. - /// An estimate of the multiplication and addition result. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 MultiplyAddEstimate(Vector256 left, Vector256 right, Vector256 addend) - { - if (Fma.IsSupported) - { - return Fma.MultiplyAdd(left, right, addend); - } - - Vector128 lower = Vector128_.MultiplyAddEstimate(left.GetLower(), right.GetLower(), addend.GetLower()); - Vector128 upper = Vector128_.MultiplyAddEstimate(left.GetUpper(), right.GetUpper(), addend.GetUpper()); - - return Vector256.Create(lower, upper); - } - - /// - /// Computes ( * ) + , rounded as one ternary operation. - /// - /// The first vector to multiply. - /// The second vector to multiply. - /// The vector to add to the product. - /// The fused multiplication and addition result. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 FusedMultiplyAdd(Vector256 left, Vector256 right, Vector256 addend) - { - if (Fma.IsSupported) - { - return Fma.MultiplyAdd(left, right, addend); - } - - // Match the runtime fallback by recursively applying the same fused contract to both halves. - Vector128 lower = Vector128_.FusedMultiplyAdd(left.GetLower(), right.GetLower(), addend.GetLower()); - Vector128 upper = Vector128_.FusedMultiplyAdd(left.GetUpper(), right.GetUpper(), addend.GetUpper()); - - return Vector256.Create(lower, upper); - } - /// /// Performs a multiplication and a negated addition of the . /// @@ -244,8 +180,8 @@ internal static class Vector256_ Vector256 min = Vector256.Create((int)ushort.MinValue); Vector256 max = Vector256.Create((int)ushort.MaxValue); - Vector256 lefClamped = Clamp(left, min, max).AsUInt32(); - Vector256 rightClamped = Clamp(right, min, max).AsUInt32(); + Vector256 lefClamped = Vector256.Clamp(left, min, max).AsUInt32(); + Vector256 rightClamped = Vector256.Clamp(right, min, max).AsUInt32(); return Vector256.Narrow(lefClamped, rightClamped); } @@ -265,8 +201,8 @@ internal static class Vector256_ Vector256 min = Vector256.Create((int)short.MinValue); Vector256 max = Vector256.Create((int)short.MaxValue); - Vector256 lefClamped = Clamp(left, min, max); - Vector256 rightClamped = Clamp(right, min, max); + Vector256 lefClamped = Vector256.Clamp(left, min, max); + Vector256 rightClamped = Vector256.Clamp(right, min, max); return Vector256.Narrow(lefClamped, rightClamped); } @@ -286,23 +222,11 @@ internal static class Vector256_ Vector256 min = Vector256.Create((short)sbyte.MinValue); Vector256 max = Vector256.Create((short)sbyte.MaxValue); - Vector256 lefClamped = Clamp(left, min, max); - Vector256 rightClamped = Clamp(right, min, max); + Vector256 lefClamped = Vector256.Clamp(left, min, max); + Vector256 rightClamped = Vector256.Clamp(right, min, max); return Vector256.Narrow(lefClamped, rightClamped); } - /// - /// Restricts a vector between a minimum and a maximum value. - /// - /// The type of the elements in the vector. - /// The vector to restrict. - /// The minimum value. - /// The maximum value. - /// The restricted . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Clamp(Vector256 value, Vector256 min, Vector256 max) - => Vector256.Min(Vector256.Max(value, min), max); - /// /// Widens a to a . /// @@ -473,56 +397,4 @@ internal static class Vector256_ return Vector256.Create(lo, hi); } - - /// - /// Subtract packed signed 16-bit integers in from packed signed 16-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed signed 16-bit integers to subtract from. - /// - /// - /// The second vector containing packed signed 16-bit integers to subtract. - /// - /// - /// A vector containing the results of subtracting packed unsigned 16-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 SubtractSaturate(Vector256 left, Vector256 right) - { - if (Avx2.IsSupported) - { - return Avx2.SubtractSaturate(left, right); - } - - // The .NET 10 portable implementation applies the same saturated operation to - // both 128-bit halves, allowing each half to select its native instruction set. - return Vector256.Create(Vector128_.SubtractSaturate(left.GetLower(), right.GetLower()), Vector128_.SubtractSaturate(left.GetUpper(), right.GetUpper())); - } - - /// - /// Subtract packed unsigned 8-bit integers in from packed unsigned 8-bit integers - /// in using saturation, and store the results. - /// - /// - /// The first vector containing packed unsigned 8-bit integers to subtract from. - /// - /// - /// The second vector containing packed unsigned 8-bit integers to subtract. - /// - /// - /// A vector containing the results of subtracting packed unsigned 8-bit integers - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 SubtractSaturate(Vector256 left, Vector256 right) - { - if (Avx2.IsSupported) - { - return Avx2.SubtractSaturate(left, right); - } - - return Vector256.Create( - Vector128_.SubtractSaturate(left.GetLower(), right.GetLower()), - Vector128_.SubtractSaturate(left.GetUpper(), right.GetUpper())); - } } diff --git a/src/ImageSharp/Common/Helpers/Vector512Utilities.cs b/src/ImageSharp/Common/Helpers/Vector512Utilities.cs index d2c5b4a87..c9d1d0da4 100644 --- a/src/ImageSharp/Common/Helpers/Vector512Utilities.cs +++ b/src/ImageSharp/Common/Helpers/Vector512Utilities.cs @@ -30,25 +30,6 @@ internal static class Vector512_ public static Vector512 ShuffleNative(Vector512 vector, [ConstantExpected] byte control) => Avx512F.Shuffle(vector, vector, control); - /// - /// Creates a new vector by selecting values from an input vector using a set of indices. - /// - /// The input vector from which values are selected. - /// - /// The per-element indices used to select a value from . - /// - /// The . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShuffleNative(Vector512 vector, Vector512 indices) - { - if (Avx512BW.IsSupported) - { - return Avx512BW.Shuffle(vector, indices); - } - - return Vector512.Shuffle(vector, indices); - } - /// /// Performs a conversion from a 512-bit vector of 16 single-precision floating-point values to a 512-bit vector of 16 signed 32-bit integer values. /// Rounding is equivalent to . @@ -72,83 +53,6 @@ internal static class Vector512_ return Avx512F.ConvertToVector512Int32WithTruncation(vector + half); } - /// - /// Rounds all values in to the nearest integer - /// following semantics. - /// - /// The vector. - /// The vector with each value rounded to the nearest integer. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 RoundToNearestInteger(Vector512 vector) - - // imm8 = 0b1000: - // imm8[7:4] = 0b0000 -> preserve 0 fractional bits (round to whole numbers) - // imm8[3:0] = 0b1000 -> _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC (round to nearest even, suppress exceptions) - => Avx512F.RoundScale(vector, 0b0000_1000); - - /// - /// Computes an estimate of ( * ) + . - /// - /// The first vector to multiply. - /// The second vector to multiply. - /// The vector to add to the product. - /// An estimate of the multiplication and addition result. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 MultiplyAddEstimate(Vector512 left, Vector512 right, Vector512 addend) - { - if (Avx512F.IsSupported) - { - return Avx512F.FusedMultiplyAdd(left, right, addend); - } - - Vector256 lower = Vector256_.MultiplyAddEstimate(left.GetLower(), right.GetLower(), addend.GetLower()); - Vector256 upper = Vector256_.MultiplyAddEstimate(left.GetUpper(), right.GetUpper(), addend.GetUpper()); - - return Vector512.Create(lower, upper); - } - - /// - /// Computes ( * ) + , rounded as one ternary operation. - /// - /// The first vector to multiply. - /// The second vector to multiply. - /// The vector to add to the product. - /// The fused multiplication and addition result. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 FusedMultiplyAdd(Vector512 left, Vector512 right, Vector512 addend) - { - if (Avx512F.IsSupported) - { - return Avx512F.FusedMultiplyAdd(left, right, addend); - } - - // Match the runtime fallback by recursively applying the same fused contract to both halves. - Vector256 lower = Vector256_.FusedMultiplyAdd(left.GetLower(), right.GetLower(), addend.GetLower()); - Vector256 upper = Vector256_.FusedMultiplyAdd(left.GetUpper(), right.GetUpper(), addend.GetUpper()); - - return Vector512.Create(lower, upper); - } - - /// - /// Subtracts packed unsigned 8-bit integers in from - /// , saturating negative lane results to zero. - /// - /// The vector from which is subtracted. - /// The vector to subtract from . - /// The element-wise saturated differences. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 SubtractSaturate(Vector512 left, Vector512 right) - { - if (Avx512BW.IsSupported) - { - return Avx512BW.SubtractSaturate(left, right); - } - - // This mirrors the .NET 10 portable implementation: recursively processing both - // 256-bit halves preserves lane order and lets each half select its available ISA. - return Vector512.Create(Vector256_.SubtractSaturate(left.GetLower(), right.GetLower()), Vector256_.SubtractSaturate(left.GetUpper(), right.GetUpper())); - } - /// /// Performs a multiplication and a negated addition of the . /// @@ -163,16 +67,4 @@ internal static class Vector512_ Vector512 vm0, Vector512 vm1) => Avx512F.FusedMultiplyAddNegated(vm0, vm1, va); - - /// - /// Restricts a vector between a minimum and a maximum value. - /// - /// The type of the elements in the vector. - /// The vector to restrict. - /// The minimum value. - /// The maximum value. - /// The restricted . - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Clamp(Vector512 value, Vector512 min, Vector512 max) - => Vector512.Min(Vector512.Max(value, min), max); } diff --git a/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector128.cs b/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector128.cs index a80dcf86e..45851e244 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector128.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector128.cs @@ -72,7 +72,7 @@ internal partial struct Block8x8F [MethodImpl(InliningOptions.ShortMethod)] private static Vector128 NormalizeVector128(Vector128 value, Vector128 off, Vector128 max) - => Vector128_.Clamp(value + off, Vector128.Zero, max); + => Vector128.Clamp(value + off, Vector128.Zero, max); private static void MultiplyIntoInt16Vector128(ref Block8x8F a, ref Block8x8F b, ref Block8x8 dest) { @@ -87,7 +87,7 @@ internal partial struct Block8x8F Vector128 left = Vector128_.ConvertToInt32RoundToEven(Unsafe.Add(ref aBase, i + 0) * Unsafe.Add(ref bBase, i + 0)); Vector128 right = Vector128_.ConvertToInt32RoundToEven(Unsafe.Add(ref aBase, i + 1) * Unsafe.Add(ref bBase, i + 1)); - Unsafe.Add(ref destBase, i / 2) = Vector128_.PackSignedSaturate(left, right); + Unsafe.Add(ref destBase, i / 2) = Vector128.NarrowWithSaturation(left, right); } } } diff --git a/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector256.cs b/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector256.cs index f16452ed5..d31d05911 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector256.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/Block8x8F.Vector256.cs @@ -96,7 +96,7 @@ internal partial struct Block8x8F [MethodImpl(InliningOptions.ShortMethod)] private static Vector256 NormalizeVector256(Vector256 value, Vector256 off, Vector256 max) - => Vector256_.Clamp(value + off, Vector256.Zero, max); + => Vector256.Clamp(value + off, Vector256.Zero, max); private static void MultiplyIntoInt16Vector256(ref Block8x8F a, ref Block8x8F b, ref Block8x8 dest) { diff --git a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.GrayScaleOperator.cs b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.GrayScaleOperator.cs index 6e5753d2f..734a36c77 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.GrayScaleOperator.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.GrayScaleOperator.cs @@ -86,7 +86,7 @@ internal abstract partial class JpegColorConverterBase public static void ConvertFromRgb(Vector128 r, Vector128 g, Vector128 b, Vector128 maximumValue, Vector128 halfValue, Vector128 scale, out Vector128 c0, out Vector128 c1, out Vector128 c2, out Vector128 c3) { // The nested estimate gives each pixel the same multiply-add grouping as the scalar Rec.601 formula. - c0 = Vector128_.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b)); + c0 = Vector128.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b)); c1 = default; c2 = default; c3 = default; @@ -97,7 +97,7 @@ internal abstract partial class JpegColorConverterBase public static void ConvertFromRgb(Vector256 r, Vector256 g, Vector256 b, Vector256 maximumValue, Vector256 halfValue, Vector256 scale, out Vector256 c0, out Vector256 c1, out Vector256 c2, out Vector256 c3) { // YMM lanes evaluate the same Rec.601 equation independently, with no horizontal lane reduction. - c0 = Vector256_.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b)); + c0 = Vector256.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b)); c1 = default; c2 = default; c3 = default; @@ -108,7 +108,7 @@ internal abstract partial class JpegColorConverterBase public static void ConvertFromRgb(Vector512 r, Vector512 g, Vector512 b, Vector512 maximumValue, Vector512 halfValue, Vector512 scale, out Vector512 c0, out Vector512 c1, out Vector512 c2, out Vector512 c3) { // ZMM lanes retain the same arithmetic order as narrower paths so only SIMD width changes. - c0 = Vector512_.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b)); + c0 = Vector512.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b)); c1 = default; c2 = default; c3 = default; diff --git a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.TiffYccKOperator.cs b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.TiffYccKOperator.cs index 90cc84048..c0479e529 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.TiffYccKOperator.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.TiffYccKOperator.cs @@ -48,9 +48,9 @@ internal abstract partial class JpegColorConverterBase Vector128 k = Vector128.One - (c3 * scale); // Four lanes apply the non-rounded YCbCr matrix before their lane-aligned K modulation. - c0 = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(YCbCrOperator.RCrMult), y) * k; - c1 = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(-YCbCrOperator.GCrMult), Vector128_.MultiplyAddEstimate(cb, Vector128.Create(-YCbCrOperator.GCbMult), y)) * k; - c2 = Vector128_.MultiplyAddEstimate(cb, Vector128.Create(YCbCrOperator.BCbMult), y) * k; + c0 = Vector128.MultiplyAddEstimate(cr, Vector128.Create(YCbCrOperator.RCrMult), y) * k; + c1 = Vector128.MultiplyAddEstimate(cr, Vector128.Create(-YCbCrOperator.GCrMult), Vector128.MultiplyAddEstimate(cb, Vector128.Create(-YCbCrOperator.GCbMult), y)) * k; + c2 = Vector128.MultiplyAddEstimate(cb, Vector128.Create(YCbCrOperator.BCbMult), y) * k; } /// @@ -63,9 +63,9 @@ internal abstract partial class JpegColorConverterBase Vector256 k = Vector256.One - (c3 * scale); // Eight lanes apply the non-rounded YCbCr matrix before their lane-aligned K modulation. - c0 = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(YCbCrOperator.RCrMult), y) * k; - c1 = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(-YCbCrOperator.GCrMult), Vector256_.MultiplyAddEstimate(cb, Vector256.Create(-YCbCrOperator.GCbMult), y)) * k; - c2 = Vector256_.MultiplyAddEstimate(cb, Vector256.Create(YCbCrOperator.BCbMult), y) * k; + c0 = Vector256.MultiplyAddEstimate(cr, Vector256.Create(YCbCrOperator.RCrMult), y) * k; + c1 = Vector256.MultiplyAddEstimate(cr, Vector256.Create(-YCbCrOperator.GCrMult), Vector256.MultiplyAddEstimate(cb, Vector256.Create(-YCbCrOperator.GCbMult), y)) * k; + c2 = Vector256.MultiplyAddEstimate(cb, Vector256.Create(YCbCrOperator.BCbMult), y) * k; } /// @@ -78,9 +78,9 @@ internal abstract partial class JpegColorConverterBase Vector512 k = Vector512.One - (c3 * scale); // Sixteen lanes apply the non-rounded YCbCr matrix before their lane-aligned K modulation. - c0 = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(YCbCrOperator.RCrMult), y) * k; - c1 = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(-YCbCrOperator.GCrMult), Vector512_.MultiplyAddEstimate(cb, Vector512.Create(-YCbCrOperator.GCbMult), y)) * k; - c2 = Vector512_.MultiplyAddEstimate(cb, Vector512.Create(YCbCrOperator.BCbMult), y) * k; + c0 = Vector512.MultiplyAddEstimate(cr, Vector512.Create(YCbCrOperator.RCrMult), y) * k; + c1 = Vector512.MultiplyAddEstimate(cr, Vector512.Create(-YCbCrOperator.GCrMult), Vector512.MultiplyAddEstimate(cb, Vector512.Create(-YCbCrOperator.GCbMult), y)) * k; + c2 = Vector512.MultiplyAddEstimate(cb, Vector512.Create(YCbCrOperator.BCbMult), y) * k; } /// @@ -129,9 +129,9 @@ internal abstract partial class JpegColorConverterBase r = (r * divisor) & nonBlack; g = (g * divisor) & nonBlack; b = (b * divisor) & nonBlack; - c0 = Vector128_.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b)) * maximumValue; - c1 = halfValue + (Vector128_.MultiplyAddEstimate(Vector128.Create(-0.168736F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(-0.331264F), g, Vector128.Create(0.5F) * b)) * maximumValue); - c2 = halfValue + (Vector128_.MultiplyAddEstimate(Vector128.Create(0.5F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(-0.418688F), g, Vector128.Create(-0.081312F) * b)) * maximumValue); + c0 = Vector128.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b)) * maximumValue; + c1 = halfValue + (Vector128.MultiplyAddEstimate(Vector128.Create(-0.168736F), r, Vector128.MultiplyAddEstimate(Vector128.Create(-0.331264F), g, Vector128.Create(0.5F) * b)) * maximumValue); + c2 = halfValue + (Vector128.MultiplyAddEstimate(Vector128.Create(0.5F), r, Vector128.MultiplyAddEstimate(Vector128.Create(-0.418688F), g, Vector128.Create(-0.081312F) * b)) * maximumValue); c3 = k * maximumValue; } @@ -151,9 +151,9 @@ internal abstract partial class JpegColorConverterBase r = (r * divisor) & nonBlack; g = (g * divisor) & nonBlack; b = (b * divisor) & nonBlack; - c0 = Vector256_.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b)) * maximumValue; - c1 = halfValue + (Vector256_.MultiplyAddEstimate(Vector256.Create(-0.168736F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(-0.331264F), g, Vector256.Create(0.5F) * b)) * maximumValue); - c2 = halfValue + (Vector256_.MultiplyAddEstimate(Vector256.Create(0.5F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(-0.418688F), g, Vector256.Create(-0.081312F) * b)) * maximumValue); + c0 = Vector256.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b)) * maximumValue; + c1 = halfValue + (Vector256.MultiplyAddEstimate(Vector256.Create(-0.168736F), r, Vector256.MultiplyAddEstimate(Vector256.Create(-0.331264F), g, Vector256.Create(0.5F) * b)) * maximumValue); + c2 = halfValue + (Vector256.MultiplyAddEstimate(Vector256.Create(0.5F), r, Vector256.MultiplyAddEstimate(Vector256.Create(-0.418688F), g, Vector256.Create(-0.081312F) * b)) * maximumValue); c3 = k * maximumValue; } @@ -173,9 +173,9 @@ internal abstract partial class JpegColorConverterBase r = (r * divisor) & nonBlack; g = (g * divisor) & nonBlack; b = (b * divisor) & nonBlack; - c0 = Vector512_.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b)) * maximumValue; - c1 = halfValue + (Vector512_.MultiplyAddEstimate(Vector512.Create(-0.168736F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(-0.331264F), g, Vector512.Create(0.5F) * b)) * maximumValue); - c2 = halfValue + (Vector512_.MultiplyAddEstimate(Vector512.Create(0.5F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(-0.418688F), g, Vector512.Create(-0.081312F) * b)) * maximumValue); + c0 = Vector512.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b)) * maximumValue; + c1 = halfValue + (Vector512.MultiplyAddEstimate(Vector512.Create(-0.168736F), r, Vector512.MultiplyAddEstimate(Vector512.Create(-0.331264F), g, Vector512.Create(0.5F) * b)) * maximumValue); + c2 = halfValue + (Vector512.MultiplyAddEstimate(Vector512.Create(0.5F), r, Vector512.MultiplyAddEstimate(Vector512.Create(-0.418688F), g, Vector512.Create(-0.081312F) * b)) * maximumValue); c3 = k * maximumValue; } } diff --git a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YCbCrOperator.cs b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YCbCrOperator.cs index e5bb6db03..e36fcf4c7 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YCbCrOperator.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YCbCrOperator.cs @@ -68,13 +68,13 @@ internal abstract partial class JpegColorConverterBase // Lanes are four independent Y/Cb/Cr samples. MultiplyAddEstimate maps to FMA where available: // R uses Cr, B uses Cb, and G subtracts both chroma contributions. Rounding occurs in the sample // domain before the common normalization scale so all precisions use integer JPEG sample semantics. - Vector128 r = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(RCrMult), y); - Vector128 g = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(-GCrMult), Vector128_.MultiplyAddEstimate(cb, Vector128.Create(-GCbMult), y)); - Vector128 b = Vector128_.MultiplyAddEstimate(cb, Vector128.Create(BCbMult), y); + Vector128 r = Vector128.MultiplyAddEstimate(cr, Vector128.Create(RCrMult), y); + Vector128 g = Vector128.MultiplyAddEstimate(cr, Vector128.Create(-GCrMult), Vector128.MultiplyAddEstimate(cb, Vector128.Create(-GCbMult), y)); + Vector128 b = Vector128.MultiplyAddEstimate(cb, Vector128.Create(BCbMult), y); - c0 = Vector128_.RoundToNearestInteger(r) * scale; - c1 = Vector128_.RoundToNearestInteger(g) * scale; - c2 = Vector128_.RoundToNearestInteger(b) * scale; + c0 = Vector128.Round(r) * scale; + c1 = Vector128.Round(g) * scale; + c2 = Vector128.Round(b) * scale; } /// @@ -88,13 +88,13 @@ internal abstract partial class JpegColorConverterBase // These eight lanes have the same layout and BT.601 arithmetic as the Vector128 overload. // Keeping an explicit overload allows the JIT to emit native YMM operations without a width // switch or decomposing the vector into smaller values. - Vector256 r = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(RCrMult), y); - Vector256 g = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(-GCrMult), Vector256_.MultiplyAddEstimate(cb, Vector256.Create(-GCbMult), y)); - Vector256 b = Vector256_.MultiplyAddEstimate(cb, Vector256.Create(BCbMult), y); + Vector256 r = Vector256.MultiplyAddEstimate(cr, Vector256.Create(RCrMult), y); + Vector256 g = Vector256.MultiplyAddEstimate(cr, Vector256.Create(-GCrMult), Vector256.MultiplyAddEstimate(cb, Vector256.Create(-GCbMult), y)); + Vector256 b = Vector256.MultiplyAddEstimate(cb, Vector256.Create(BCbMult), y); - c0 = Vector256_.RoundToNearestInteger(r) * scale; - c1 = Vector256_.RoundToNearestInteger(g) * scale; - c2 = Vector256_.RoundToNearestInteger(b) * scale; + c0 = Vector256.Round(r) * scale; + c1 = Vector256.Round(g) * scale; + c2 = Vector256.Round(b) * scale; } /// @@ -108,13 +108,13 @@ internal abstract partial class JpegColorConverterBase // Sixteen independent samples occupy the ZMM lanes. The explicit constants are broadcasts; // assembly inspection verifies the JIT hoists them from the loop and retains fused operations. // The formula and rounding order remain identical to the narrower overloads. - Vector512 r = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(RCrMult), y); - Vector512 g = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(-GCrMult), Vector512_.MultiplyAddEstimate(cb, Vector512.Create(-GCbMult), y)); - Vector512 b = Vector512_.MultiplyAddEstimate(cb, Vector512.Create(BCbMult), y); + Vector512 r = Vector512.MultiplyAddEstimate(cr, Vector512.Create(RCrMult), y); + Vector512 g = Vector512.MultiplyAddEstimate(cr, Vector512.Create(-GCrMult), Vector512.MultiplyAddEstimate(cb, Vector512.Create(-GCbMult), y)); + Vector512 b = Vector512.MultiplyAddEstimate(cb, Vector512.Create(BCbMult), y); - c0 = Vector512_.RoundToNearestInteger(r) * scale; - c1 = Vector512_.RoundToNearestInteger(g) * scale; - c2 = Vector512_.RoundToNearestInteger(b) * scale; + c0 = Vector512.Round(r) * scale; + c1 = Vector512.Round(g) * scale; + c2 = Vector512.Round(b) * scale; } /// @@ -137,9 +137,9 @@ internal abstract partial class JpegColorConverterBase // Each vector holds four consecutive values from one RGB plane. The nested multiply-add sequence // produces four Y lanes, four Cb lanes, and four Cr lanes without transposition. The association // exposes two FMA opportunities per output while preserving the scalar formula's term grouping. - c0 = Vector128_.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b)); - c1 = halfValue + Vector128_.MultiplyAddEstimate(Vector128.Create(-0.168736F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(-0.331264F), g, Vector128.Create(0.5F) * b)); - c2 = halfValue + Vector128_.MultiplyAddEstimate(Vector128.Create(0.5F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(-0.418688F), g, Vector128.Create(-0.081312F) * b)); + c0 = Vector128.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b)); + c1 = halfValue + Vector128.MultiplyAddEstimate(Vector128.Create(-0.168736F), r, Vector128.MultiplyAddEstimate(Vector128.Create(-0.331264F), g, Vector128.Create(0.5F) * b)); + c2 = halfValue + Vector128.MultiplyAddEstimate(Vector128.Create(0.5F), r, Vector128.MultiplyAddEstimate(Vector128.Create(-0.418688F), g, Vector128.Create(-0.081312F) * b)); c3 = default; } @@ -149,9 +149,9 @@ internal abstract partial class JpegColorConverterBase { // Eight planar RGB samples use the identical association as Vector128, allowing direct YMM FMA // generation while preserving the component-per-vector output layout. - c0 = Vector256_.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b)); - c1 = halfValue + Vector256_.MultiplyAddEstimate(Vector256.Create(-0.168736F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(-0.331264F), g, Vector256.Create(0.5F) * b)); - c2 = halfValue + Vector256_.MultiplyAddEstimate(Vector256.Create(0.5F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(-0.418688F), g, Vector256.Create(-0.081312F) * b)); + c0 = Vector256.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b)); + c1 = halfValue + Vector256.MultiplyAddEstimate(Vector256.Create(-0.168736F), r, Vector256.MultiplyAddEstimate(Vector256.Create(-0.331264F), g, Vector256.Create(0.5F) * b)); + c2 = halfValue + Vector256.MultiplyAddEstimate(Vector256.Create(0.5F), r, Vector256.MultiplyAddEstimate(Vector256.Create(-0.418688F), g, Vector256.Create(-0.081312F) * b)); c3 = default; } @@ -161,9 +161,9 @@ internal abstract partial class JpegColorConverterBase { // Sixteen planar RGB samples use the same nested form. Constants are lane broadcasts and c3 is // deliberately zero because the shared traversal removes the unused fourth store for this operator. - c0 = Vector512_.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b)); - c1 = halfValue + Vector512_.MultiplyAddEstimate(Vector512.Create(-0.168736F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(-0.331264F), g, Vector512.Create(0.5F) * b)); - c2 = halfValue + Vector512_.MultiplyAddEstimate(Vector512.Create(0.5F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(-0.418688F), g, Vector512.Create(-0.081312F) * b)); + c0 = Vector512.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b)); + c1 = halfValue + Vector512.MultiplyAddEstimate(Vector512.Create(-0.168736F), r, Vector512.MultiplyAddEstimate(Vector512.Create(-0.331264F), g, Vector512.Create(0.5F) * b)); + c2 = halfValue + Vector512.MultiplyAddEstimate(Vector512.Create(0.5F), r, Vector512.MultiplyAddEstimate(Vector512.Create(-0.418688F), g, Vector512.Create(-0.081312F) * b)); c3 = default; } } diff --git a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YccKOperator.cs b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YccKOperator.cs index 90bee0c94..1a3b0fbb9 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YccKOperator.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YccKOperator.cs @@ -46,12 +46,12 @@ internal abstract partial class JpegColorConverterBase Vector128 scaledK = c3 * scale * scale; // Four lanes reconstruct YCbCr concurrently; each rounded result is inverted and modulated by its K lane. - Vector128 r = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(YCbCrOperator.RCrMult), y); - Vector128 g = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(-YCbCrOperator.GCrMult), Vector128_.MultiplyAddEstimate(cb, Vector128.Create(-YCbCrOperator.GCbMult), y)); - Vector128 b = Vector128_.MultiplyAddEstimate(cb, Vector128.Create(YCbCrOperator.BCbMult), y); - c0 = (maximumValue - Vector128_.RoundToNearestInteger(r)) * scaledK; - c1 = (maximumValue - Vector128_.RoundToNearestInteger(g)) * scaledK; - c2 = (maximumValue - Vector128_.RoundToNearestInteger(b)) * scaledK; + Vector128 r = Vector128.MultiplyAddEstimate(cr, Vector128.Create(YCbCrOperator.RCrMult), y); + Vector128 g = Vector128.MultiplyAddEstimate(cr, Vector128.Create(-YCbCrOperator.GCrMult), Vector128.MultiplyAddEstimate(cb, Vector128.Create(-YCbCrOperator.GCbMult), y)); + Vector128 b = Vector128.MultiplyAddEstimate(cb, Vector128.Create(YCbCrOperator.BCbMult), y); + c0 = (maximumValue - Vector128.Round(r)) * scaledK; + c1 = (maximumValue - Vector128.Round(g)) * scaledK; + c2 = (maximumValue - Vector128.Round(b)) * scaledK; } /// @@ -64,12 +64,12 @@ internal abstract partial class JpegColorConverterBase Vector256 scaledK = c3 * scale * scale; // Eight lanes retain planar alignment from Y/Cb/Cr/K through normalized RGB. - Vector256 r = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(YCbCrOperator.RCrMult), y); - Vector256 g = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(-YCbCrOperator.GCrMult), Vector256_.MultiplyAddEstimate(cb, Vector256.Create(-YCbCrOperator.GCbMult), y)); - Vector256 b = Vector256_.MultiplyAddEstimate(cb, Vector256.Create(YCbCrOperator.BCbMult), y); - c0 = (maximumValue - Vector256_.RoundToNearestInteger(r)) * scaledK; - c1 = (maximumValue - Vector256_.RoundToNearestInteger(g)) * scaledK; - c2 = (maximumValue - Vector256_.RoundToNearestInteger(b)) * scaledK; + Vector256 r = Vector256.MultiplyAddEstimate(cr, Vector256.Create(YCbCrOperator.RCrMult), y); + Vector256 g = Vector256.MultiplyAddEstimate(cr, Vector256.Create(-YCbCrOperator.GCrMult), Vector256.MultiplyAddEstimate(cb, Vector256.Create(-YCbCrOperator.GCbMult), y)); + Vector256 b = Vector256.MultiplyAddEstimate(cb, Vector256.Create(YCbCrOperator.BCbMult), y); + c0 = (maximumValue - Vector256.Round(r)) * scaledK; + c1 = (maximumValue - Vector256.Round(g)) * scaledK; + c2 = (maximumValue - Vector256.Round(b)) * scaledK; } /// @@ -82,12 +82,12 @@ internal abstract partial class JpegColorConverterBase Vector512 scaledK = c3 * scale * scale; // Sixteen lanes use the same matrix, rounding, inversion, and K modulation order as scalar code. - Vector512 r = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(YCbCrOperator.RCrMult), y); - Vector512 g = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(-YCbCrOperator.GCrMult), Vector512_.MultiplyAddEstimate(cb, Vector512.Create(-YCbCrOperator.GCbMult), y)); - Vector512 b = Vector512_.MultiplyAddEstimate(cb, Vector512.Create(YCbCrOperator.BCbMult), y); - c0 = (maximumValue - Vector512_.RoundToNearestInteger(r)) * scaledK; - c1 = (maximumValue - Vector512_.RoundToNearestInteger(g)) * scaledK; - c2 = (maximumValue - Vector512_.RoundToNearestInteger(b)) * scaledK; + Vector512 r = Vector512.MultiplyAddEstimate(cr, Vector512.Create(YCbCrOperator.RCrMult), y); + Vector512 g = Vector512.MultiplyAddEstimate(cr, Vector512.Create(-YCbCrOperator.GCrMult), Vector512.MultiplyAddEstimate(cb, Vector512.Create(-YCbCrOperator.GCbMult), y)); + Vector512 b = Vector512.MultiplyAddEstimate(cb, Vector512.Create(YCbCrOperator.BCbMult), y); + c0 = (maximumValue - Vector512.Round(r)) * scaledK; + c1 = (maximumValue - Vector512.Round(g)) * scaledK; + c2 = (maximumValue - Vector512.Round(b)) * scaledK; } /// diff --git a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverterBase.Icc.cs b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverterBase.Icc.cs index 37f1297a6..4b04a653d 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverterBase.Icc.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverterBase.Icc.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Numerics; +using System.Numerics.Tensors; using System.Runtime.InteropServices; using SixLabors.ImageSharp.ColorProfiles; using SixLabors.ImageSharp.ColorProfiles.Icc; @@ -42,7 +43,7 @@ internal abstract partial class JpegColorConverterBase { // The single luminance plane is the ICC source, so it is normalized in place. The temporary // buffer is still RGB-sized because the profile conversion expands each Y sample to three lanes. - TensorPrimitives_.Multiply(c0, 1F / this.MaximumValue, c0); + TensorPrimitives.Multiply(c0, 1F / this.MaximumValue, c0); Span source = MemoryMarshal.Cast(c0); Span destination = MemoryMarshal.Cast(packed); diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs index c68dd19b4..3a17e8064 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using System.Numerics; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -118,7 +119,7 @@ internal class ComponentProcessor : IDisposable static void SumVertical(Span target, Span source) // Exact destination overlap is supported, so each accumulated row remains in target. - => TensorPrimitives_.Add(target, source, target); + => TensorPrimitives.Add(target, source, target); static void SumHorizontal(Span target, int factor) { @@ -168,6 +169,6 @@ internal class ComponentProcessor : IDisposable static void MultiplyToAverage(Span target, float multiplier) // Apply the subsampling reciprocal in place after all contributing rows have been summed. - => TensorPrimitives_.Multiply(target, multiplier, target); + => TensorPrimitives.Multiply(target, multiplier, target); } } diff --git a/src/ImageSharp/Formats/Jpeg/Components/FloatingPointDCT.Vector256.cs b/src/ImageSharp/Formats/Jpeg/Components/FloatingPointDCT.Vector256.cs index 7c342d7a0..ae7a0626b 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/FloatingPointDCT.Vector256.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/FloatingPointDCT.Vector256.cs @@ -55,8 +55,8 @@ internal static partial class FloatingPointDCT tmp12 = tmp6 + tmp7; Vector256 z5 = (tmp10 - tmp12) * Vector256.Create(0.382683433f); // mm256_F_0_3826 - Vector256 z2 = Vector256_.MultiplyAddEstimate(Vector256.Create(0.541196100f), tmp10, z5); // mm256_F_0_5411 - Vector256 z4 = Vector256_.MultiplyAddEstimate(Vector256.Create(1.306562965f), tmp12, z5); // mm256_F_1_3065 + Vector256 z2 = Vector256.MultiplyAddEstimate(Vector256.Create(0.541196100f), tmp10, z5); // mm256_F_0_5411 + Vector256 z4 = Vector256.MultiplyAddEstimate(Vector256.Create(1.306562965f), tmp12, z5); // mm256_F_1_3065 Vector256 z3 = tmp11 * mm256_F_0_7071; Vector256 z11 = tmp7 + z3; @@ -122,8 +122,8 @@ internal static partial class FloatingPointDCT z5 = (z10 + z12) * Vector256.Create(1.847759065f); // mm256_F_1_8477 - tmp10 = Vector256_.MultiplyAddEstimate(z12, Vector256.Create(-1.082392200f), z5); // mm256_F_n1_0823 - tmp12 = Vector256_.MultiplyAddEstimate(z10, Vector256.Create(-2.613125930f), z5); // mm256_F_n2_6131 + tmp10 = Vector256.MultiplyAddEstimate(z12, Vector256.Create(-1.082392200f), z5); // mm256_F_n1_0823 + tmp12 = Vector256.MultiplyAddEstimate(z10, Vector256.Create(-2.613125930f), z5); // mm256_F_n2_6131 tmp6 = tmp12 - tmp7; tmp5 = tmp11 - tmp6; diff --git a/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs b/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs index bae1ac477..4fc3baaae 100644 --- a/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs +++ b/src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs @@ -286,10 +286,10 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator // distanceAbove = |left - upperLeft| // Computing both unsigned subtraction directions and OR-ing them obtains // each absolute difference without widening the byte lanes. - Vector128 aboveMinusUpper = Vector128_.SubtractSaturate(above, upperLeft); - Vector128 leftMinusUpper = Vector128_.SubtractSaturate(left, upperLeft); - Vector128 distanceLeft = Vector128_.SubtractSaturate(upperLeft, above) | aboveMinusUpper; - Vector128 distanceAbove = Vector128_.SubtractSaturate(upperLeft, left) | leftMinusUpper; + Vector128 aboveMinusUpper = Vector128.SubtractSaturate(above, upperLeft); + Vector128 leftMinusUpper = Vector128.SubtractSaturate(left, upperLeft); + Vector128 distanceLeft = Vector128.SubtractSaturate(upperLeft, above) | aboveMinusUpper; + Vector128 distanceAbove = Vector128.SubtractSaturate(upperLeft, left) | leftMinusUpper; return SelectPredictor(left, above, upperLeft, aboveMinusUpper, leftMinusUpper, distanceLeft, distanceAbove); } @@ -307,10 +307,10 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator // Apply the same Paeth identities as the 128-bit path to thirty-two lanes. // Saturating subtraction in both directions forms the absolute differences // without widening, preserving one predictor result per source byte. - Vector256 aboveMinusUpper = Vector256_.SubtractSaturate(above, upperLeft); - Vector256 leftMinusUpper = Vector256_.SubtractSaturate(left, upperLeft); - Vector256 distanceLeft = Vector256_.SubtractSaturate(upperLeft, above) | aboveMinusUpper; - Vector256 distanceAbove = Vector256_.SubtractSaturate(upperLeft, left) | leftMinusUpper; + Vector256 aboveMinusUpper = Vector256.SubtractSaturate(above, upperLeft); + Vector256 leftMinusUpper = Vector256.SubtractSaturate(left, upperLeft); + Vector256 distanceLeft = Vector256.SubtractSaturate(upperLeft, above) | aboveMinusUpper; + Vector256 distanceAbove = Vector256.SubtractSaturate(upperLeft, left) | leftMinusUpper; return SelectPredictor(left, above, upperLeft, aboveMinusUpper, leftMinusUpper, distanceLeft, distanceAbove); } @@ -328,10 +328,10 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator // Apply the same byte-lane Paeth identities to sixty-four AVX-512BW lanes. // No cross-lane operation is required because every component has its own // left, above, and upper-left inputs at the matching vector index. - Vector512 aboveMinusUpper = Vector512_.SubtractSaturate(above, upperLeft); - Vector512 leftMinusUpper = Vector512_.SubtractSaturate(left, upperLeft); - Vector512 distanceLeft = Vector512_.SubtractSaturate(upperLeft, above) | aboveMinusUpper; - Vector512 distanceAbove = Vector512_.SubtractSaturate(upperLeft, left) | leftMinusUpper; + Vector512 aboveMinusUpper = Vector512.SubtractSaturate(above, upperLeft); + Vector512 leftMinusUpper = Vector512.SubtractSaturate(left, upperLeft); + Vector512 distanceLeft = Vector512.SubtractSaturate(upperLeft, above) | aboveMinusUpper; + Vector512 distanceAbove = Vector512.SubtractSaturate(upperLeft, left) | leftMinusUpper; return SelectPredictor(left, above, upperLeft, aboveMinusUpper, leftMinusUpper, distanceLeft, distanceAbove); } @@ -356,7 +356,7 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator // their summed distance and cannot beat either neighbor; the all-bits mask // excludes upper-left. On opposite sides, that distance is the absolute // difference between distanceLeft and distanceAbove. - Vector128 distanceUpper = sameDirection | Vector128_.SubtractSaturate(distanceAbove, distanceLeft) | Vector128_.SubtractSaturate(distanceLeft, distanceAbove); + Vector128 distanceUpper = sameDirection | Vector128.SubtractSaturate(distanceAbove, distanceLeft) | Vector128.SubtractSaturate(distanceLeft, distanceAbove); // Equality selects above before upper-left, implementing PNG's second tie rule. Vector128 minimumAboveUpper = Vector128.Min(distanceUpper, distanceAbove); @@ -384,7 +384,7 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator // Exclude upper-left when its distance is the non-minimal sum; otherwise // compute its distance as the absolute difference of the two known distances. - Vector256 distanceUpper = sameDirection | Vector256_.SubtractSaturate(distanceAbove, distanceLeft) | Vector256_.SubtractSaturate(distanceLeft, distanceAbove); + Vector256 distanceUpper = sameDirection | Vector256.SubtractSaturate(distanceAbove, distanceLeft) | Vector256.SubtractSaturate(distanceLeft, distanceAbove); // Select above on equality, then select left on equality to preserve PNG's // required left, above, upper-left tie order in every byte lane. @@ -412,7 +412,7 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator // Exclude upper-left when its distance is the non-minimal sum; otherwise // compute its distance as the absolute difference of the two known distances. - Vector512 distanceUpper = sameDirection | Vector512_.SubtractSaturate(distanceAbove, distanceLeft) | Vector512_.SubtractSaturate(distanceLeft, distanceAbove); + Vector512 distanceUpper = sameDirection | Vector512.SubtractSaturate(distanceAbove, distanceLeft) | Vector512.SubtractSaturate(distanceLeft, distanceAbove); // Select above on equality, then select left on equality to preserve PNG's // required left, above, upper-left tie order in every byte lane. diff --git a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs index d64a1ea51..5368c5da4 100644 --- a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs +++ b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Common.Helpers; @@ -24,7 +25,7 @@ internal static class UpFilter DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); // The leading filter byte is metadata; every remaining byte is the modulo-256 sum of Raw(x) and Prior(x). - TensorPrimitives_.Add(scanline[1..], previousScanline[1..], scanline[1..]); + TensorPrimitives.Add(scanline[1..], previousScanline[1..], scanline[1..]); } /// diff --git a/src/ImageSharp/Formats/Png/PngCgbiProcessor.cs b/src/ImageSharp/Formats/Png/PngCgbiProcessor.cs index 3addbb3c0..a19125f3b 100644 --- a/src/ImageSharp/Formats/Png/PngCgbiProcessor.cs +++ b/src/ImageSharp/Formats/Png/PngCgbiProcessor.cs @@ -113,7 +113,7 @@ internal static class PngCgbiProcessor { ref byte blockRef = ref Unsafe.Add(ref scanlineRef, i * Unsafe.SizeOf()); Vector512 bgra = Unsafe.ReadUnaligned>(ref blockRef); - Vector512 rgba = Vector512_.ShuffleNative(bgra, shuffleMask); + Vector512 rgba = Vector512.ShuffleNative(bgra, shuffleMask); Vector512 packed = rgba.AsInt32(); Vector512 alpha = Vector512.ShiftRightLogical(packed, 24); @@ -254,7 +254,7 @@ internal static class PngCgbiProcessor { ref byte blockRef = ref Unsafe.Add(ref scanlineRef, i * Unsafe.SizeOf()); Vector128 bgra = Unsafe.ReadUnaligned>(ref blockRef); - Vector128 rgba = Vector128_.ShuffleNative(bgra, shuffleMask); + Vector128 rgba = Vector128.ShuffleNative(bgra, shuffleMask); Vector128 packed = rgba.AsInt32(); Vector128 alpha = Vector128.ShiftRightLogical(packed, 24); diff --git a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs index 9ea040182..1c41dfbb4 100644 --- a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs +++ b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -364,7 +365,7 @@ internal class AlphaDecoder : IDisposable else { // Byte addition intentionally wraps modulo 256, matching the WebP alpha predictor. - TensorPrimitives_.Add(input[..width], prev[..width], dst[..width]); + TensorPrimitives.Add(input[..width], prev[..width], dst[..width]); } } diff --git a/src/ImageSharp/Formats/Webp/Lossless/ColorSpaceTransformUtils.cs b/src/ImageSharp/Formats/Webp/Lossless/ColorSpaceTransformUtils.cs index 10a27b382..347dcf9a9 100644 --- a/src/ImageSharp/Formats/Webp/Lossless/ColorSpaceTransformUtils.cs +++ b/src/ImageSharp/Formats/Webp/Lossless/ColorSpaceTransformUtils.cs @@ -86,8 +86,8 @@ internal static class ColorSpaceTransformUtils nuint input1Idx = x + (span / 2); Vector128 input0 = Unsafe.As>(ref Unsafe.Add(ref inputRef, input0Idx)).AsByte(); Vector128 input1 = Unsafe.As>(ref Unsafe.Add(ref inputRef, input1Idx)).AsByte(); - Vector128 r0 = Vector128_.ShuffleNative(input0, collectColorBlueTransformsShuffleLowMask); - Vector128 r1 = Vector128_.ShuffleNative(input1, collectColorBlueTransformsShuffleHighMask); + Vector128 r0 = Vector128.ShuffleNative(input0, collectColorBlueTransformsShuffleLowMask); + Vector128 r1 = Vector128.ShuffleNative(input1, collectColorBlueTransformsShuffleHighMask); Vector128 r = r0 | r1; Vector128 gb0 = input0 & collectColorBlueTransformsGreenBlueMask; Vector128 gb1 = input1 & collectColorBlueTransformsGreenBlueMask; diff --git a/src/ImageSharp/Formats/Webp/Lossless/LosslessUtils.cs b/src/ImageSharp/Formats/Webp/Lossless/LosslessUtils.cs index ed89d0e94..37bd4945b 100644 --- a/src/ImageSharp/Formats/Webp/Lossless/LosslessUtils.cs +++ b/src/ImageSharp/Formats/Webp/Lossless/LosslessUtils.cs @@ -128,7 +128,7 @@ internal static unsafe class LosslessUtils { ref uint pos = ref Unsafe.Add(ref MemoryMarshal.GetReference(pixelData), i); Vector128 input = Unsafe.As>(ref pos).AsByte(); - Vector128 in0g0g = Vector128_.ShuffleNative(input, addGreenToBlueAndRedMask); + Vector128 in0g0g = Vector128.ShuffleNative(input, addGreenToBlueAndRedMask); Vector128 output = input + in0g0g; Unsafe.As>(ref pos) = output.AsUInt32(); i += 4; @@ -192,7 +192,7 @@ internal static unsafe class LosslessUtils { ref uint pos = ref Unsafe.Add(ref MemoryMarshal.GetReference(pixelData), i); Vector128 input = Unsafe.As>(ref pos).AsByte(); - Vector128 in0g0g = Vector128_.ShuffleNative(input, subtractGreenFromBlueAndRedMask); + Vector128 in0g0g = Vector128.ShuffleNative(input, subtractGreenFromBlueAndRedMask); Vector128 output = input - in0g0g; Unsafe.As>(ref pos) = output.AsUInt32(); i += 4; @@ -387,7 +387,7 @@ internal static unsafe class LosslessUtils Vector128 b = Vector128_.ShuffleLow(a.AsInt16(), SimdUtils.Shuffle.MMShuffle2200); Vector128 c = Vector128_.ShuffleHigh(b.AsInt16(), SimdUtils.Shuffle.MMShuffle2200); Vector128 d = Vector128_.MultiplyHigh(c.AsInt16(), multsrb.AsInt16()); - Vector128 e = Vector128_.ShiftLeftLogical(input.AsInt16(), 8); + Vector128 e = input.AsInt16() << 8; Vector128 f = Vector128_.MultiplyHigh(e.AsInt16(), multsb2.AsInt16()); Vector128 g = Vector128.ShiftRightLogical(f.AsInt32(), 16); Vector128 h = g.AsByte() + d.AsByte(); @@ -479,7 +479,7 @@ internal static unsafe class LosslessUtils Vector128 c = Vector128_.ShuffleHigh(b.AsInt16(), SimdUtils.Shuffle.MMShuffle2200); Vector128 d = Vector128_.MultiplyHigh(c.AsInt16(), multsrb.AsInt16()); Vector128 e = input.AsByte() + d.AsByte(); - Vector128 f = Vector128_.ShiftLeftLogical(e.AsInt16(), 8); + Vector128 f = e.AsInt16() << 8; Vector128 g = Vector128_.MultiplyHigh(f, multsb2.AsInt16()); Vector128 h = Vector128.ShiftRightLogical(g.AsInt32(), 8); Vector128 i = h.AsByte() + f.AsByte(); @@ -1442,10 +1442,10 @@ internal static unsafe class LosslessUtils Vector128 a0 = Vector128.CreateScalar(a).AsByte(); Vector128 b0 = Vector128.CreateScalar(b).AsByte(); Vector128 c0 = Vector128.CreateScalar(c).AsByte(); - Vector128 ac0 = Vector128_.SubtractSaturate(a0, c0); - Vector128 ca0 = Vector128_.SubtractSaturate(c0, a0); - Vector128 bc0 = Vector128_.SubtractSaturate(b0, c0); - Vector128 cb0 = Vector128_.SubtractSaturate(c0, b0); + Vector128 ac0 = Vector128.SubtractSaturate(a0, c0); + Vector128 ca0 = Vector128.SubtractSaturate(c0, a0); + Vector128 bc0 = Vector128.SubtractSaturate(b0, c0); + Vector128 cb0 = Vector128.SubtractSaturate(c0, b0); Vector128 ac = ac0 | ca0; Vector128 bc = bc0 | cb0; Vector128 pa = Vector128_.UnpackLow(ac, Vector128.Zero); // |a - c| diff --git a/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs b/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs index 7c8ce400a..355018649 100644 --- a/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs +++ b/src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using System.Buffers; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Memory; @@ -331,7 +332,7 @@ internal abstract unsafe class Vp8LHistogram { if (b.IsUsed(0)) { - TensorPrimitives_.Add(this.Literal[..literalSize], b.Literal[..literalSize], output.Literal[..literalSize]); + TensorPrimitives.Add(this.Literal[..literalSize], b.Literal[..literalSize], output.Literal[..literalSize]); } else { @@ -354,7 +355,7 @@ internal abstract unsafe class Vp8LHistogram { if (b.IsUsed(1)) { - TensorPrimitives_.Add(this.Red[..size], b.Red[..size], output.Red[..size]); + TensorPrimitives.Add(this.Red[..size], b.Red[..size], output.Red[..size]); } else { @@ -377,7 +378,7 @@ internal abstract unsafe class Vp8LHistogram { if (b.IsUsed(2)) { - TensorPrimitives_.Add(this.Blue[..size], b.Blue[..size], output.Blue[..size]); + TensorPrimitives.Add(this.Blue[..size], b.Blue[..size], output.Blue[..size]); } else { @@ -400,7 +401,7 @@ internal abstract unsafe class Vp8LHistogram { if (b.IsUsed(3)) { - TensorPrimitives_.Add(this.Alpha[..size], b.Alpha[..size], output.Alpha[..size]); + TensorPrimitives.Add(this.Alpha[..size], b.Alpha[..size], output.Alpha[..size]); } else { @@ -423,7 +424,7 @@ internal abstract unsafe class Vp8LHistogram { if (b.IsUsed(4)) { - TensorPrimitives_.Add(this.Distance[..size], b.Distance[..size], output.Distance[..size]); + TensorPrimitives.Add(this.Distance[..size], b.Distance[..size], output.Distance[..size]); } else { diff --git a/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs b/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs index a58494dd4..d4d1997f0 100644 --- a/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs +++ b/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs @@ -77,7 +77,7 @@ internal static class LossyUtils Vector256 b01s = Vector256_.UnpackLow(b01.AsByte(), Vector256.Zero); // subtract, square and accumulate. - Vector256 d0 = Vector256_.SubtractSaturate(a01s.AsInt16(), b01s.AsInt16()); + Vector256 d0 = Vector256.SubtractSaturate(a01s.AsInt16(), b01s.AsInt16()); Vector256 e0 = Vector256_.MultiplyAddAdjacent(d0, d0); return ReduceSumVector256(e0); @@ -110,8 +110,8 @@ internal static class LossyUtils Vector128 b23s = Vector128_.UnpackLow(b23.AsByte(), Vector128.Zero); // subtract, square and accumulate. - Vector128 d0 = Vector128_.SubtractSaturate(a01s.AsInt16(), b01s.AsInt16()); - Vector128 d1 = Vector128_.SubtractSaturate(a23s.AsInt16(), b23s.AsInt16()); + Vector128 d0 = Vector128.SubtractSaturate(a01s.AsInt16(), b01s.AsInt16()); + Vector128 d1 = Vector128.SubtractSaturate(a23s.AsInt16(), b23s.AsInt16()); Vector128 e0 = Vector128_.MultiplyAddAdjacent(d0, d0); Vector128 e1 = Vector128_.MultiplyAddAdjacent(d1, d1); Vector128 sum = e0 + e1; @@ -203,8 +203,8 @@ internal static class LossyUtils private static Vector128 SubtractAndAccumulateVector128(Vector128 a, Vector128 b) { // Take abs(a-b) in 8b. - Vector128 ab = Vector128_.SubtractSaturate(a, b); - Vector128 ba = Vector128_.SubtractSaturate(b, a); + Vector128 ab = Vector128.SubtractSaturate(a, b); + Vector128 ba = Vector128.SubtractSaturate(b, a); Vector128 absAb = ab | ba; // Zero-extend to 16b. @@ -222,8 +222,8 @@ internal static class LossyUtils private static Vector256 SubtractAndAccumulateVector256(Vector256 a, Vector256 b) { // Take abs(a-b) in 8b. - Vector256 ab = Vector256_.SubtractSaturate(a, b); - Vector256 ba = Vector256_.SubtractSaturate(b, a); + Vector256 ab = Vector256.SubtractSaturate(a, b); + Vector256 ba = Vector256.SubtractSaturate(b, a); Vector256 absAb = ab | ba; // Zero-extend to 16b. @@ -1985,20 +1985,20 @@ internal static class LossyUtils q0 ^= signBit; q1 ^= signBit; - Vector128 t1 = Vector128_.SubtractSaturate(p1.AsSByte(), q1.AsSByte()); // p1 - q1 + Vector128 t1 = Vector128.SubtractSaturate(p1.AsSByte(), q1.AsSByte()); // p1 - q1 t1 = (~notHev & t1.AsByte()).AsSByte(); // hev(p1 - q1) - Vector128 t2 = Vector128_.SubtractSaturate(q0.AsSByte(), p0.AsSByte()); // q0 - p0 - t1 = Vector128_.AddSaturate(t1, t2); // hev(p1 - q1) + 1 * (q0 - p0) - t1 = Vector128_.AddSaturate(t1, t2); // hev(p1 - q1) + 2 * (q0 - p0) - t1 = Vector128_.AddSaturate(t1, t2); // hev(p1 - q1) + 3 * (q0 - p0) + Vector128 t2 = Vector128.SubtractSaturate(q0.AsSByte(), p0.AsSByte()); // q0 - p0 + t1 = Vector128.AddSaturate(t1, t2); // hev(p1 - q1) + 1 * (q0 - p0) + t1 = Vector128.AddSaturate(t1, t2); // hev(p1 - q1) + 2 * (q0 - p0) + t1 = Vector128.AddSaturate(t1, t2); // hev(p1 - q1) + 3 * (q0 - p0) t1 = (t1.AsByte() & mask).AsSByte(); // mask filter values we don't care about. - t2 = Vector128_.AddSaturate(t1, Vector128.Create((byte)3).AsSByte()); // 3 * (q0 - p0) + hev(p1 - q1) + 3 - Vector128 t3 = Vector128_.AddSaturate(t1, Vector128.Create((byte)4).AsSByte()); // 3 * (q0 - p0) + hev(p1 - q1) + 4 + t2 = Vector128.AddSaturate(t1, Vector128.Create((byte)3).AsSByte()); // 3 * (q0 - p0) + hev(p1 - q1) + 3 + Vector128 t3 = Vector128.AddSaturate(t1, Vector128.Create((byte)4).AsSByte()); // 3 * (q0 - p0) + hev(p1 - q1) + 4 t2 = SignedShift8bVector128(t2.AsByte()); // (3 * (q0 - p0) + hev(p1 - q1) + 3) >> 3 t3 = SignedShift8bVector128(t3.AsByte()); // (3 * (q0 - p0) + hev(p1 - q1) + 4) >> 3 - p0 = Vector128_.AddSaturate(p0.AsSByte(), t2).AsByte(); // p0 += t2 - q0 = Vector128_.SubtractSaturate(q0.AsSByte(), t3).AsByte(); // q0 -= t3 + p0 = Vector128.AddSaturate(p0.AsSByte(), t2).AsByte(); // p0 += t2 + q0 = Vector128.SubtractSaturate(q0.AsSByte(), t3).AsByte(); // q0 -= t3 p0 ^= signBit; q0 ^= signBit; @@ -2008,8 +2008,8 @@ internal static class LossyUtils t3 -= Vector128.Create((sbyte)64); t3 = (notHev & t3.AsByte()).AsSByte(); // if !hev - q1 = Vector128_.SubtractSaturate(q1.AsSByte(), t3).AsByte(); // q1 -= t3 - p1 = Vector128_.AddSaturate(p1.AsSByte(), t3).AsByte(); // p1 += t3 + q1 = Vector128.SubtractSaturate(q1.AsSByte(), t3).AsByte(); // q1 -= t3 + p1 = Vector128.AddSaturate(p1.AsSByte(), t3).AsByte(); // p1 += t3 p1 = p1.AsByte() ^ signBit; q1 = q1.AsByte() ^ signBit; } @@ -2063,13 +2063,13 @@ internal static class LossyUtils private static void DoSimpleFilterVector128(ref Vector128 p0, ref Vector128 q0, Vector128 fl) { - Vector128 v3 = Vector128_.AddSaturate(fl.AsSByte(), Vector128.Create((byte)3).AsSByte()); - Vector128 v4 = Vector128_.AddSaturate(fl.AsSByte(), Vector128.Create((byte)4).AsSByte()); + Vector128 v3 = Vector128.AddSaturate(fl.AsSByte(), Vector128.Create((byte)3).AsSByte()); + Vector128 v4 = Vector128.AddSaturate(fl.AsSByte(), Vector128.Create((byte)4).AsSByte()); v4 = SignedShift8bVector128(v4.AsByte()).AsSByte(); // v4 >> 3 v3 = SignedShift8bVector128(v3.AsByte()).AsSByte(); // v3 >> 3 - q0 = Vector128_.SubtractSaturate(q0.AsSByte(), v4).AsByte(); // q0 -= v4 - p0 = Vector128_.AddSaturate(p0.AsSByte(), v3).AsByte(); // p0 += v3 + q0 = Vector128.SubtractSaturate(q0.AsSByte(), v4).AsByte(); // q0 -= v4 + p0 = Vector128.AddSaturate(p0.AsSByte(), v3).AsByte(); // p0 += v3 } private static Vector128 GetNotHevVector128(ref Vector128 p1, ref Vector128 p0, ref Vector128 q0, ref Vector128 q1, int hevThresh) @@ -2080,7 +2080,7 @@ internal static class LossyUtils Vector128 h = Vector128.Create((byte)hevThresh); Vector128 tMax = Vector128.Max(t1, t2); - Vector128 tMaxH = Vector128_.SubtractSaturate(tMax, h); + Vector128 tMaxH = Vector128.SubtractSaturate(tMax, h); // not_hev <= t1 && not_hev <= t2 return Vector128.Equals(tMaxH, Vector128.Zero); @@ -2173,10 +2173,10 @@ internal static class LossyUtils Vector128 t3 = Vector128.ShiftRightLogical(t2.AsInt16(), 1); // abs(p1 - q1) / 2 Vector128 t4 = AbsVector128(p0, q0); // abs(p0 - q0) - Vector128 t5 = Vector128_.AddSaturate(t4, t4); // abs(p0 - q0) * 2 - Vector128 t6 = Vector128_.AddSaturate(t5.AsByte(), t3.AsByte()); // abs(p0-q0)*2 + abs(p1-q1)/2 + Vector128 t5 = Vector128.AddSaturate(t4, t4); // abs(p0 - q0) * 2 + Vector128 t6 = Vector128.AddSaturate(t5.AsByte(), t3.AsByte()); // abs(p0-q0)*2 + abs(p1-q1)/2 - Vector128 t7 = Vector128_.SubtractSaturate(t6, mthresh.AsByte()); // mask <= m_thresh + Vector128 t7 = Vector128.SubtractSaturate(t6, mthresh.AsByte()); // mask <= m_thresh return Vector128.Equals(t7, Vector128.Zero); } @@ -2290,11 +2290,11 @@ internal static class LossyUtils private static Vector128 GetBaseDeltaVector128(Vector128 p1, Vector128 p0, Vector128 q0, Vector128 q1) { // Beware of addition order, for saturation! - Vector128 p1q1 = Vector128_.SubtractSaturate(p1, q1); // p1 - q1 - Vector128 q0p0 = Vector128_.SubtractSaturate(q0, p0); // q0 - p0 - Vector128 s1 = Vector128_.AddSaturate(p1q1, q0p0); // p1 - q1 + 1 * (q0 - p0) - Vector128 s2 = Vector128_.AddSaturate(q0p0, s1); // p1 - q1 + 2 * (q0 - p0) - return Vector128_.AddSaturate(q0p0, s2); // p1 - q1 + 3 * (q0 - p0) + Vector128 p1q1 = Vector128.SubtractSaturate(p1, q1); // p1 - q1 + Vector128 q0p0 = Vector128.SubtractSaturate(q0, p0); // q0 - p0 + Vector128 s1 = Vector128.AddSaturate(p1q1, q0p0); // p1 - q1 + 1 * (q0 - p0) + Vector128 s2 = Vector128.AddSaturate(q0p0, s1); // p1 - q1 + 2 * (q0 - p0) + return Vector128.AddSaturate(q0p0, s2); // p1 - q1 + 3 * (q0 - p0) } // Shift each byte of "x" by 3 bits while preserving by the sign bit. @@ -2306,14 +2306,14 @@ internal static class LossyUtils Vector128 low1 = Vector128.ShiftRightArithmetic(low0.AsInt16(), 3 + 8); Vector128 high1 = Vector128.ShiftRightArithmetic(high0.AsInt16(), 3 + 8); - return Vector128_.PackSignedSaturate(low1, high1); + return Vector128.NarrowWithSaturation(low1, high1); } [MethodImpl(InliningOptions.ShortMethod)] private static void ComplexMaskVector128(Vector128 p1, Vector128 p0, Vector128 q0, Vector128 q1, int thresh, int ithresh, ref Vector128 mask) { Vector128 it = Vector128.Create((byte)ithresh); - Vector128 diff = Vector128_.SubtractSaturate(mask, it); + Vector128 diff = Vector128.SubtractSaturate(mask, it); Vector128 threshMask = Vector128.Equals(diff, Vector128.Zero); Vector128 filterMask = NeedsFilterVector128(p1, p0, q0, q1, thresh); @@ -2329,9 +2329,9 @@ internal static class LossyUtils Vector128 signBit = Vector128.Create((byte)0x80); Vector128 a1Low = Vector128.ShiftRightArithmetic(a0Low, 7); Vector128 a1High = Vector128.ShiftRightArithmetic(a0High, 7); - Vector128 delta = Vector128_.PackSignedSaturate(a1Low, a1High); - pi = Vector128_.AddSaturate(pi.AsSByte(), delta).AsByte(); - qi = Vector128_.SubtractSaturate(qi.AsSByte(), delta).AsByte(); + Vector128 delta = Vector128.NarrowWithSaturation(a1Low, a1High); + pi = Vector128.AddSaturate(pi.AsSByte(), delta).AsByte(); + qi = Vector128.SubtractSaturate(qi.AsSByte(), delta).AsByte(); pi ^= signBit.AsByte(); qi ^= signBit.AsByte(); } @@ -2354,7 +2354,7 @@ internal static class LossyUtils // Compute abs(p - q) = subs(p - q) OR subs(q - p) [MethodImpl(InliningOptions.ShortMethod)] private static Vector128 AbsVector128(Vector128 p, Vector128 q) - => Vector128_.SubtractSaturate(q, p) | Vector128_.SubtractSaturate(p, q); + => Vector128.SubtractSaturate(q, p) | Vector128.SubtractSaturate(p, q); [MethodImpl(InliningOptions.ShortMethod)] private static bool Hev(ReadOnlySpan p, int offset, int step, int thresh) diff --git a/src/ImageSharp/Formats/Webp/Lossy/Vp8Encoding.cs b/src/ImageSharp/Formats/Webp/Lossy/Vp8Encoding.cs index 0130e0799..077d43d5a 100644 --- a/src/ImageSharp/Formats/Webp/Lossy/Vp8Encoding.cs +++ b/src/ImageSharp/Formats/Webp/Lossy/Vp8Encoding.cs @@ -542,8 +542,8 @@ internal static unsafe class Vp8Encoding Vector128 tmp32 = tmp31 + Vector128.Create(937); Vector128 tmp1 = Vector128.ShiftRightArithmetic(tmp12, 9); Vector128 tmp3 = Vector128.ShiftRightArithmetic(tmp32, 9); - Vector128 s03 = Vector128_.PackSignedSaturate(tmp0, tmp2); - Vector128 s12 = Vector128_.PackSignedSaturate(tmp1, tmp3); + Vector128 s03 = Vector128.NarrowWithSaturation(tmp0, tmp2); + Vector128 s12 = Vector128.NarrowWithSaturation(tmp1, tmp3); Vector128 slo = Vector128_.UnpackLow(s03, s12); // 0 1 0 1 0 1... Vector128 shi = Vector128_.UnpackHigh(s03, s12); // 2 3 2 3 2 3 Vector128 v23 = Vector128_.UnpackHigh(slo.AsInt32(), shi.AsInt32()); @@ -569,8 +569,8 @@ internal static unsafe class Vp8Encoding // f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16) // f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16) - Vector128 f1 = Vector128_.PackSignedSaturate(e1, e1); - Vector128 f3 = Vector128_.PackSignedSaturate(e3, e3); + Vector128 f1 = Vector128.NarrowWithSaturation(e1, e1); + Vector128 f3 = Vector128.NarrowWithSaturation(e3, e3); // g1 = f1 + (a3 != 0); // The compare will return (0xffff, 0) for (==0, !=0). To turn that into the diff --git a/src/ImageSharp/Formats/Webp/Lossy/YuvConversion.cs b/src/ImageSharp/Formats/Webp/Lossy/YuvConversion.cs index 11c969660..d533f1d5f 100644 --- a/src/ImageSharp/Formats/Webp/Lossy/YuvConversion.cs +++ b/src/ImageSharp/Formats/Webp/Lossy/YuvConversion.cs @@ -686,12 +686,12 @@ internal static class YuvConversion out Vector128 output4, out Vector128 output5) { - output0 = Vector128_.ShuffleNative(input0, shuffle0); - output1 = Vector128_.ShuffleNative(input0, shuffle1); - output2 = Vector128_.ShuffleNative(input0, shuffle2); - output3 = Vector128_.ShuffleNative(input1, shuffle0); - output4 = Vector128_.ShuffleNative(input1, shuffle1); - output5 = Vector128_.ShuffleNative(input1, shuffle2); + output0 = Vector128.ShuffleNative(input0, shuffle0); + output1 = Vector128.ShuffleNative(input0, shuffle1); + output2 = Vector128.ShuffleNative(input0, shuffle2); + output3 = Vector128.ShuffleNative(input1, shuffle0); + output4 = Vector128.ShuffleNative(input1, shuffle1); + output5 = Vector128.ShuffleNative(input1, shuffle2); } // Convert 32 samples of YUV444 to B/G/R @@ -726,8 +726,8 @@ internal static class YuvConversion Vector128 g4 = g2 - g3; Vector128 b0 = Vector128_.MultiplyHigh(u0.AsUInt16(), Vector128.Create(26, 129, 26, 129, 26, 129, 26, 129, 26, 129, 26, 129, 26, 129, 26, 129).AsUInt16()); - Vector128 b1 = Vector128_.AddSaturate(b0, y1); - Vector128 b2 = Vector128_.SubtractSaturate(b1, Vector128.Create((ushort)17685)); + Vector128 b1 = Vector128.AddSaturate(b0, y1); + Vector128 b2 = Vector128.SubtractSaturate(b1, Vector128.Create((ushort)17685)); // Use logical shift for B2, which can be larger than 32767. r = Vector128.ShiftRightArithmetic(r2.AsInt16(), 6); // range: [-14234, 30815] diff --git a/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs b/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs index acfa26b4f..b59150190 100644 --- a/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs +++ b/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs @@ -134,9 +134,9 @@ internal static class WebpCommonUtils Vector128 b1 = (a1 & alphaMask).AsInt32(); Vector128 b2 = (a2 & alphaMask).AsInt32(); Vector128 b3 = (a3 & alphaMask).AsInt32(); - Vector128 c0 = Vector128_.PackSignedSaturate(b0, b1).AsInt16(); - Vector128 c1 = Vector128_.PackSignedSaturate(b2, b3).AsInt16(); - Vector128 d = Vector128_.PackSignedSaturate(c0, c1).AsByte(); + Vector128 c0 = Vector128.NarrowWithSaturation(b0, b1).AsInt16(); + Vector128 c1 = Vector128.NarrowWithSaturation(b2, b3).AsInt16(); + Vector128 d = Vector128.NarrowWithSaturation(c0, c1).AsByte(); Vector128 bits = Vector128.Equals(d, Vector128.Create((byte)0x80).AsByte()); uint mask = bits.ExtractMostSignificantBits(); return mask != 0xFFFF; @@ -150,8 +150,8 @@ internal static class WebpCommonUtils Vector128 a1 = Vector128.Load(src + i + 16).AsByte(); Vector128 b0 = (a0 & alphaMask).AsInt32(); Vector128 b1 = (a1 & alphaMask).AsInt32(); - Vector128 c = Vector128_.PackSignedSaturate(b0, b1).AsInt16(); - Vector128 d = Vector128_.PackSignedSaturate(c, c).AsByte(); + Vector128 c = Vector128.NarrowWithSaturation(b0, b1).AsInt16(); + Vector128 d = Vector128.NarrowWithSaturation(c, c).AsByte(); Vector128 bits = Vector128.Equals(d, Vector128.Create((byte)0x80).AsByte()); uint mask = bits.ExtractMostSignificantBits(); return mask != 0xFFFF; diff --git a/src/ImageSharp/ImageSharp.csproj b/src/ImageSharp/ImageSharp.csproj index a05d243f9..971d73b84 100644 --- a/src/ImageSharp/ImageSharp.csproj +++ b/src/ImageSharp/ImageSharp.csproj @@ -47,7 +47,8 @@ - + + diff --git a/src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPorterDuffFunctions.cs b/src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPorterDuffFunctions.cs index 893e8e41a..3d7175dcb 100644 --- a/src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPorterDuffFunctions.cs +++ b/src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPorterDuffFunctions.cs @@ -848,7 +848,7 @@ internal static partial class AssociatedAlphaPorterDuffFunctions Vector4 sourceAlpha = Numerics.PermuteW(source); Vector4 destinationAlpha = Numerics.PermuteW(destination); Vector4 coefficient = Vector4.One - sourceAlpha; - Vector4 result = Vector128_.FusedMultiplyAdd(destination.AsVector128(), coefficient.AsVector128(), overlap.AsVector128()).AsVector4(); + Vector4 result = Vector128.FusedMultiplyAdd(destination.AsVector128(), coefficient.AsVector128(), overlap.AsVector128()).AsVector4(); return Numerics.WithW(result, destinationAlpha); } @@ -859,7 +859,7 @@ internal static partial class AssociatedAlphaPorterDuffFunctions { Vector256 sourceAlpha = Avx.Permute(source, ShuffleAlphaControl); Vector256 destinationAlpha = Avx.Permute(destination, ShuffleAlphaControl); - Vector256 result = Vector256_.FusedMultiplyAdd(destination, Vector256.Create(1F) - sourceAlpha, overlap); + Vector256 result = Vector256.FusedMultiplyAdd(destination, Vector256.Create(1F) - sourceAlpha, overlap); return Avx.Blend(result, destinationAlpha, BlendAlphaControl); } @@ -870,7 +870,7 @@ internal static partial class AssociatedAlphaPorterDuffFunctions { Vector512 sourceAlpha = Vector512_.ShuffleNative(source, ShuffleAlphaControl); Vector512 destinationAlpha = Vector512_.ShuffleNative(destination, ShuffleAlphaControl); - Vector512 result = Vector512_.FusedMultiplyAdd(destination, Vector512.Create(1F) - sourceAlpha, overlap); + Vector512 result = Vector512.FusedMultiplyAdd(destination, Vector512.Create(1F) - sourceAlpha, overlap); return Vector512.ConditionalSelect(AlphaMask512(), destinationAlpha, result); } @@ -959,18 +959,18 @@ internal static partial class AssociatedAlphaPorterDuffFunctions public static Vector4 BlendWithCoverage(Vector4 backdrop, Vector4 source, float coverage) { // Use the same fused operation as the wider paths so exact midpoints cannot change across vector widths. - return Vector128_.FusedMultiplyAdd((source - backdrop).AsVector128(), Vector128.Create(coverage), backdrop.AsVector128()).AsVector4(); + return Vector128.FusedMultiplyAdd((source - backdrop).AsVector128(), Vector128.Create(coverage), backdrop.AsVector128()).AsVector4(); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector256 BlendWithCoverage(Vector256 backdrop, Vector256 source, Vector256 coverage) - => Vector256_.FusedMultiplyAdd(source - backdrop, coverage, backdrop); + => Vector256.FusedMultiplyAdd(source - backdrop, coverage, backdrop); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector512 BlendWithCoverage(Vector512 backdrop, Vector512 source, Vector512 coverage) - => Vector512_.FusedMultiplyAdd(source - backdrop, coverage, backdrop); + => Vector512.FusedMultiplyAdd(source - backdrop, coverage, backdrop); /// /// Calculates one associated Overlay overlap component without recovering either straight component. diff --git a/src/ImageSharp/PixelFormats/PixelBlenders/PorterDuffFunctions.cs b/src/ImageSharp/PixelFormats/PixelBlenders/PorterDuffFunctions.cs index 4b048a6f4..c319b9662 100644 --- a/src/ImageSharp/PixelFormats/PixelBlenders/PorterDuffFunctions.cs +++ b/src/ImageSharp/PixelFormats/PixelBlenders/PorterDuffFunctions.cs @@ -586,7 +586,7 @@ internal static partial class PorterDuffFunctions Vector4 sourcePremultiplied = Numerics.WithW(source * sourceAlpha, sourceAlpha); // Use the same fused operation as the wider paths so exact midpoints cannot change across vector widths. - Vector4 result = Vector128_.MultiplyAddEstimate((sourcePremultiplied - backdropPremultiplied).AsVector128(), Vector128.Create(coverage), backdropPremultiplied.AsVector128()).AsVector4(); + Vector4 result = Vector128.MultiplyAddEstimate((sourcePremultiplied - backdropPremultiplied).AsVector128(), Vector128.Create(coverage), backdropPremultiplied.AsVector128()).AsVector4(); Numerics.UnPremultiply(ref result); return result; @@ -606,7 +606,7 @@ internal static partial class PorterDuffFunctions Vector256 sourceAlpha = Avx.Permute(source, ShuffleAlphaControl); Vector256 backdropPremultiplied = Avx.Blend(backdrop * backdropAlpha, backdropAlpha, BlendAlphaControl); Vector256 sourcePremultiplied = Avx.Blend(source * sourceAlpha, sourceAlpha, BlendAlphaControl); - Vector256 result = Vector256_.MultiplyAddEstimate(sourcePremultiplied - backdropPremultiplied, coverage, backdropPremultiplied); + Vector256 result = Vector256.MultiplyAddEstimate(sourcePremultiplied - backdropPremultiplied, coverage, backdropPremultiplied); return Numerics.UnPremultiply(result, Avx.Permute(result, ShuffleAlphaControl)); } @@ -626,7 +626,7 @@ internal static partial class PorterDuffFunctions Vector512 alphaMask = AlphaMask512(); Vector512 backdropPremultiplied = Vector512.ConditionalSelect(alphaMask, backdropAlpha, backdrop * backdropAlpha); Vector512 sourcePremultiplied = Vector512.ConditionalSelect(alphaMask, sourceAlpha, source * sourceAlpha); - Vector512 result = Vector512_.MultiplyAddEstimate(sourcePremultiplied - backdropPremultiplied, coverage, backdropPremultiplied); + Vector512 result = Vector512.MultiplyAddEstimate(sourcePremultiplied - backdropPremultiplied, coverage, backdropPremultiplied); return Numerics.UnPremultiply(result, Vector512_.ShuffleNative(result, ShuffleAlphaControl)); } @@ -836,8 +836,8 @@ internal static partial class PorterDuffFunctions // calculate final color Vector256 color = destination * dstW; - color = Vector256_.MultiplyAddEstimate(source, srcW, color); - color = Vector256_.MultiplyAddEstimate(blend, blendW, color); + color = Vector256.MultiplyAddEstimate(source, srcW, color); + color = Vector256.MultiplyAddEstimate(blend, blendW, color); // unpremultiply return Numerics.UnPremultiply(color, alpha); @@ -866,8 +866,8 @@ internal static partial class PorterDuffFunctions // calculate final color Vector512 color = destination * dstW; - color = Vector512_.MultiplyAddEstimate(source, srcW, color); - color = Vector512_.MultiplyAddEstimate(blend, blendW, color); + color = Vector512.MultiplyAddEstimate(source, srcW, color); + color = Vector512.MultiplyAddEstimate(blend, blendW, color); // unpremultiply return Numerics.UnPremultiply(color, alpha); @@ -920,7 +920,7 @@ internal static partial class PorterDuffFunctions Vector256 dstW = alpha - blendW; // calculate final color - Vector256 color = Vector256_.MultiplyAddEstimate(destination, dstW, Avx.Multiply(blend, blendW)); + Vector256 color = Vector256.MultiplyAddEstimate(destination, dstW, Avx.Multiply(blend, blendW)); // unpremultiply return Numerics.UnPremultiply(color, alpha); @@ -945,7 +945,7 @@ internal static partial class PorterDuffFunctions Vector512 dstW = alpha - blendW; // calculate final color - Vector512 color = Vector512_.MultiplyAddEstimate(destination, dstW, blend * blendW); + Vector512 color = Vector512.MultiplyAddEstimate(destination, dstW, blend * blendW); // unpremultiply return Numerics.UnPremultiply(color, alpha); @@ -1104,8 +1104,8 @@ internal static partial class PorterDuffFunctions Vector256 dstW = vOne - sW; // calculate alpha - Vector256 alpha = Vector256_.MultiplyAddEstimate(sW, srcW, Avx.Multiply(dW, dstW)); - Vector256 color = Vector256_.MultiplyAddEstimate(Avx.Multiply(sW, source), srcW, Avx.Multiply(Avx.Multiply(dW, destination), dstW)); + Vector256 alpha = Vector256.MultiplyAddEstimate(sW, srcW, Avx.Multiply(dW, dstW)); + Vector256 color = Vector256.MultiplyAddEstimate(Avx.Multiply(sW, source), srcW, Avx.Multiply(Avx.Multiply(dW, destination), dstW)); // unpremultiply return Numerics.UnPremultiply(color, alpha); @@ -1129,8 +1129,8 @@ internal static partial class PorterDuffFunctions Vector512 dstW = vOne - sW; // calculate alpha - Vector512 alpha = Vector512_.MultiplyAddEstimate(sW, srcW, dW * dstW); - Vector512 color = Vector512_.MultiplyAddEstimate(sW * source, srcW, (dW * destination) * dstW); + Vector512 alpha = Vector512.MultiplyAddEstimate(sW, srcW, dW * dstW); + Vector512 color = Vector512.MultiplyAddEstimate(sW * source, srcW, (dW * destination) * dstW); // unpremultiply return Numerics.UnPremultiply(color, alpha); diff --git a/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/NormalizedByte4P.PixelOperations.cs b/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/NormalizedByte4P.PixelOperations.cs index de2f8f17b..5928cbe52 100644 --- a/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/NormalizedByte4P.PixelOperations.cs +++ b/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/NormalizedByte4P.PixelOperations.cs @@ -451,7 +451,7 @@ public partial struct NormalizedByte4P source = Vector512.Min(Vector512.Max(source, zero), one); Vector512 alpha = Vector512_.ShuffleNative(source, 0b_11_11_11_11); Vector512 nativeAlpha = Vector512.Min(Vector512.Max((alpha * Vector512.Create(2F)) - one, -one), one); - Vector512 storedAlpha = Vector512_.RoundToNearestInteger(nativeAlpha * Vector512.Create(MaxPos)); + Vector512 storedAlpha = Vector512.Round(nativeAlpha * Vector512.Create(MaxPos)); storedAlpha += Vector512.Create(MaxPos); storedAlpha /= Vector512.Create(ScaledMagnitude); Vector512 result = source * storedAlpha; @@ -480,7 +480,7 @@ public partial struct NormalizedByte4P source = Vector256.Min(Vector256.Max(source, zero), one); Vector256 alpha = Vector256_.ShuffleNative(source, 0b_11_11_11_11); Vector256 nativeAlpha = Vector256.Min(Vector256.Max((alpha * Vector256.Create(2F)) - one, -one), one); - Vector256 storedAlpha = Vector256_.RoundToNearestInteger(nativeAlpha * Vector256.Create(MaxPos)); + Vector256 storedAlpha = Vector256.Round(nativeAlpha * Vector256.Create(MaxPos)); storedAlpha += Vector256.Create(MaxPos); storedAlpha /= Vector256.Create(ScaledMagnitude); Vector256 result = source * storedAlpha; @@ -509,7 +509,7 @@ public partial struct NormalizedByte4P source = Vector128.Min(Vector128.Max(source, zero), one); Vector128 alpha = Vector128_.ShuffleNative(source, 0b_11_11_11_11); Vector128 nativeAlpha = Vector128.Min(Vector128.Max((alpha * Vector128.Create(2F)) - one, -one), one); - Vector128 storedAlpha = Vector128_.RoundToNearestInteger(nativeAlpha * Vector128.Create(MaxPos)); + Vector128 storedAlpha = Vector128.Round(nativeAlpha * Vector128.Create(MaxPos)); storedAlpha += Vector128.Create(MaxPos); storedAlpha /= Vector128.Create(ScaledMagnitude); Vector128 result = source * storedAlpha; @@ -610,7 +610,7 @@ public partial struct NormalizedByte4P Vector512 alpha = Vector512_.ShuffleNative(source, 0b_11_11_11_11); Vector512 nativeAlpha = Vector512.Min(Vector512.Max((alpha * Vector512.Create(2F)) - one, -one), one); - Vector512 storedAlpha = Vector512_.RoundToNearestInteger(nativeAlpha * Vector512.Create(MaxPos)); + Vector512 storedAlpha = Vector512.Round(nativeAlpha * Vector512.Create(MaxPos)); storedAlpha += Vector512.Create(MaxPos); storedAlpha /= Vector512.Create(ScaledMagnitude); Vector512 result = source * (storedAlpha / alpha); @@ -640,7 +640,7 @@ public partial struct NormalizedByte4P Vector256 alpha = Vector256_.ShuffleNative(source, 0b_11_11_11_11); Vector256 nativeAlpha = Vector256.Min(Vector256.Max((alpha * Vector256.Create(2F)) - one, -one), one); - Vector256 storedAlpha = Vector256_.RoundToNearestInteger(nativeAlpha * Vector256.Create(MaxPos)); + Vector256 storedAlpha = Vector256.Round(nativeAlpha * Vector256.Create(MaxPos)); storedAlpha += Vector256.Create(MaxPos); storedAlpha /= Vector256.Create(ScaledMagnitude); Vector256 result = source * (storedAlpha / alpha); @@ -670,7 +670,7 @@ public partial struct NormalizedByte4P Vector128 alpha = Vector128_.ShuffleNative(source, 0b_11_11_11_11); Vector128 nativeAlpha = Vector128.Min(Vector128.Max((alpha * Vector128.Create(2F)) - one, -one), one); - Vector128 storedAlpha = Vector128_.RoundToNearestInteger(nativeAlpha * Vector128.Create(MaxPos)); + Vector128 storedAlpha = Vector128.Round(nativeAlpha * Vector128.Create(MaxPos)); storedAlpha += Vector128.Create(MaxPos); storedAlpha /= Vector128.Create(ScaledMagnitude); Vector128 result = source * (storedAlpha / alpha); @@ -701,7 +701,7 @@ public partial struct NormalizedByte4P Vector512 vector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, (uint)i)); Vector512 integers = ConvertToPackedInt32(vector, scaled); Vector256 shorts = Vector256_.PackSignedSaturate(integers.GetLower(), integers.GetUpper()); - Vector128 packed = Vector128_.PackSignedSaturate(shorts.GetLower(), shorts.GetUpper()); + Vector128 packed = Vector128.NarrowWithSaturation(shorts.GetLower(), shorts.GetUpper()); if (Avx2.IsSupported) { @@ -721,8 +721,8 @@ public partial struct NormalizedByte4P { Vector256 vector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, (uint)i)); Vector256 integers = ConvertToPackedInt32(vector, scaled); - Vector128 shorts = Vector128_.PackSignedSaturate(integers.GetLower(), integers.GetUpper()); - Vector128 packed = Vector128_.PackSignedSaturate(shorts, shorts); + Vector128 shorts = Vector128.NarrowWithSaturation(integers.GetLower(), integers.GetUpper()); + Vector128 packed = Vector128.NarrowWithSaturation(shorts, shorts); Unsafe.As(ref Unsafe.Add(ref destinationBase, (uint)i)) = packed.AsUInt64().GetElement(0); } } @@ -733,8 +733,8 @@ public partial struct NormalizedByte4P { Vector128 vector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, (uint)i)); Vector128 integers = ConvertToPackedInt32(vector, scaled); - Vector128 shorts = Vector128_.PackSignedSaturate(integers, integers); - Vector128 packed = Vector128_.PackSignedSaturate(shorts, shorts); + Vector128 shorts = Vector128.NarrowWithSaturation(integers, integers); + Vector128 packed = Vector128.NarrowWithSaturation(shorts, shorts); Unsafe.Add(ref destinationBase, (uint)i).PackedValue = packed.AsUInt32().GetElement(0); } diff --git a/src/ImageSharp/PixelFormats/Utils/SignedShort4PixelOperations.cs b/src/ImageSharp/PixelFormats/Utils/SignedShort4PixelOperations.cs index 4bf8c5e26..22d1ba4ac 100644 --- a/src/ImageSharp/PixelFormats/Utils/SignedShort4PixelOperations.cs +++ b/src/ImageSharp/PixelFormats/Utils/SignedShort4PixelOperations.cs @@ -149,7 +149,7 @@ internal static class SignedShort4PixelOperations { Vector256 vectors = Unsafe.As>(ref Unsafe.Add(ref sourceBase, (uint)index)); Vector256 integers = ConvertToInt32(vectors, normalized, scaled); - Vector128 packed = Vector128_.PackSignedSaturate(integers.GetLower(), integers.GetUpper()); + Vector128 packed = Vector128.NarrowWithSaturation(integers.GetLower(), integers.GetUpper()); Unsafe.As>(ref Unsafe.Add(ref destinationBase, (uint)(index * Vector128.Count))) = packed; } } @@ -162,7 +162,7 @@ internal static class SignedShort4PixelOperations { Vector128 vector = Unsafe.As>(ref Unsafe.Add(ref sourceBase, (uint)index)); Vector128 integers = ConvertToInt32(vector, normalized, scaled); - Vector128 packed = Vector128_.PackSignedSaturate(integers, integers); + Vector128 packed = Vector128.NarrowWithSaturation(integers, integers); Unsafe.WriteUnaligned(ref Unsafe.Add(ref destinationBytes, (uint)(index * sizeof(ulong))), packed.AsUInt64().GetElement(0)); } diff --git a/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs b/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs index db9eff55c..df76fd081 100644 --- a/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs +++ b/src/ImageSharp/Processing/Processors/Convolution/ConvolutionProcessorHelpers.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using System.Diagnostics.CodeAnalysis; +using System.Numerics.Tensors; using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.Processing.Processors.Convolution; @@ -38,7 +39,7 @@ internal static class ConvolutionProcessorHelpers } // Divide every weight by the accumulated Gaussian sum so the kernel has unit response. - TensorPrimitives_.Divide(kernel, sum, kernel); + TensorPrimitives.Divide(kernel, sum, kernel); return kernel; } @@ -67,14 +68,14 @@ internal static class ConvolutionProcessorHelpers // Invert the kernel for sharpening. int midpointRounded = (int)midpoint; float midpointValue = kernel[midpointRounded]; - TensorPrimitives_.Negate(kernel, kernel); + TensorPrimitives.Negate(kernel, kernel); // The sharpening kernel negates every Gaussian weight except its center. Restore that original // center while adding twice the Gaussian sum so the complete kernel retains unit response. kernel[midpointRounded] = (2F * sum) - midpointValue; // Sharpening changes signs but preserves the Gaussian sum, so the same divisor produces unit response. - TensorPrimitives_.Divide(kernel, sum, kernel); + TensorPrimitives.Divide(kernel, sum, kernel); return kernel; } diff --git a/src/ImageSharp/Processing/Processors/Convolution/Parameters/BokehBlurKernelDataProvider.cs b/src/ImageSharp/Processing/Processors/Convolution/Parameters/BokehBlurKernelDataProvider.cs index 12f6e55d5..1593dc788 100644 --- a/src/ImageSharp/Processing/Processors/Convolution/Parameters/BokehBlurKernelDataProvider.cs +++ b/src/ImageSharp/Processing/Processors/Convolution/Parameters/BokehBlurKernelDataProvider.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Numerics; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SixLabors.ImageSharp.Common.Helpers; @@ -214,7 +215,7 @@ internal static class BokehBlurKernelDataProvider // Complex64 stores each value as adjacent real and imaginary floats. Multiplying that flattened // float span scales both parts independently, which is exactly complex multiplication by a real scalar. Span values = MemoryMarshal.Cast(kernelsRef.AsSpan()); - TensorPrimitives_.Multiply(values, scalar, values); + TensorPrimitives.Multiply(values, scalar, values); } } } diff --git a/src/ImageSharp/Processing/Processors/Normalization/HistogramEqualizationProcessor{TPixel}.cs b/src/ImageSharp/Processing/Processors/Normalization/HistogramEqualizationProcessor{TPixel}.cs index b624d5b0e..77b741e88 100644 --- a/src/ImageSharp/Processing/Processors/Normalization/HistogramEqualizationProcessor{TPixel}.cs +++ b/src/ImageSharp/Processing/Processors/Normalization/HistogramEqualizationProcessor{TPixel}.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using System.Numerics; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SixLabors.ImageSharp.Common.Helpers; @@ -116,7 +117,7 @@ internal abstract class HistogramEqualizationProcessor : ImageProcessor< int addToEachBin = sumOverClip > 0 ? (int)MathF.Floor(sumOverClip / this.luminanceLevelsFloat) : 0; if (addToEachBin > 0) { - TensorPrimitives_.Add(histogram, addToEachBin, histogram); + TensorPrimitives.Add(histogram, addToEachBin, histogram); } int residual = sumOverClip - (addToEachBin * this.LuminanceLevels); diff --git a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernel.cs b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernel.cs index aa24878c6..e30640ad1 100644 --- a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernel.cs +++ b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernel.cs @@ -104,8 +104,8 @@ internal readonly unsafe struct ResizeKernel Vector256 pixels256_0 = Unsafe.As>(ref rowStartRef); Vector256 pixels256_1 = Unsafe.As>(ref Unsafe.Add(ref rowStartRef, (nuint)2)); - result256_0 = Vector256_.MultiplyAddEstimate(Vector256.Load(bufferStart), pixels256_0, result256_0); - result256_1 = Vector256_.MultiplyAddEstimate(Vector256.Load(bufferStart + 8), pixels256_1, result256_1); + result256_0 = Vector256.MultiplyAddEstimate(Vector256.Load(bufferStart), pixels256_0, result256_0); + result256_1 = Vector256.MultiplyAddEstimate(Vector256.Load(bufferStart + 8), pixels256_1, result256_1); bufferStart += 16; rowStartRef = ref Unsafe.Add(ref rowStartRef, (nuint)4); @@ -116,7 +116,7 @@ internal readonly unsafe struct ResizeKernel if ((this.Length & 3) >= 2) { Vector256 pixels256_0 = Unsafe.As>(ref rowStartRef); - result256_0 = Vector256_.MultiplyAddEstimate(Vector256.Load(bufferStart), pixels256_0, result256_0); + result256_0 = Vector256.MultiplyAddEstimate(Vector256.Load(bufferStart), pixels256_0, result256_0); bufferStart += 8; rowStartRef = ref Unsafe.Add(ref rowStartRef, (nuint)2); @@ -127,7 +127,7 @@ internal readonly unsafe struct ResizeKernel if ((this.Length & 1) != 0) { Vector128 pixels128 = Unsafe.As>(ref rowStartRef); - result128 = Vector128_.MultiplyAddEstimate(Vector128.Load(bufferStart), pixels128, result128); + result128 = Vector128.MultiplyAddEstimate(Vector128.Load(bufferStart), pixels128, result128); } return result128.AsVector4(); diff --git a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs index 0b8106e0b..b397b9999 100644 --- a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs +++ b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Diagnostics; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SixLabors.ImageSharp.Memory; @@ -243,7 +244,7 @@ internal partial class ResizeKernelMap : IDisposable // Normalize, best to do it here rather than in the pixel loop later on. if (sum > 0) { - Numerics.Normalize(kernelValues, sum); + TensorPrimitives.Divide(kernelValues, sum, kernelValues); } kernel.FillOrCopyAndExpand(kernelValues); diff --git a/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj b/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj index e060e1f33..c3733c73c 100644 --- a/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj +++ b/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj @@ -69,7 +69,6 @@ - diff --git a/tests/ImageSharp.Tests/Common/NumericsTests.cs b/tests/ImageSharp.Tests/Common/NumericsTests.cs index 280522c16..6f35118e2 100644 --- a/tests/ImageSharp.Tests/Common/NumericsTests.cs +++ b/tests/ImageSharp.Tests/Common/NumericsTests.cs @@ -53,4 +53,5 @@ public class NumericsTests Assert.True(expected == actual, $"IsOutOfRange({value}, {min}, {max})"); } + } diff --git a/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs b/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs index 0334d166f..b4fbf5fd8 100644 --- a/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs +++ b/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs @@ -151,9 +151,9 @@ public partial class SimdUtilsTests private static void RunVectorMultiplyAddUsesRuntimeOrderAndFusedContract() { - Assert.Equal(Vector128.Create(11F), Vector128_.MultiplyAddEstimate(Vector128.Create(2F), Vector128.Create(3F), Vector128.Create(5F))); - Assert.Equal(Vector256.Create(11F), Vector256_.MultiplyAddEstimate(Vector256.Create(2F), Vector256.Create(3F), Vector256.Create(5F))); - Assert.Equal(Vector512.Create(11F), Vector512_.MultiplyAddEstimate(Vector512.Create(2F), Vector512.Create(3F), Vector512.Create(5F))); + Assert.Equal(Vector128.Create(11F), Vector128.MultiplyAddEstimate(Vector128.Create(2F), Vector128.Create(3F), Vector128.Create(5F))); + Assert.Equal(Vector256.Create(11F), Vector256.MultiplyAddEstimate(Vector256.Create(2F), Vector256.Create(3F), Vector256.Create(5F))); + Assert.Equal(Vector512.Create(11F), Vector512.MultiplyAddEstimate(Vector512.Create(2F), Vector512.Create(3F), Vector512.Create(5F))); // These associated-alpha components produce an exact midpoint that a separate multiply and add rounds incorrectly. float left = (68F / byte.MaxValue) * .625F; @@ -161,9 +161,9 @@ public partial class SimdUtilsTests float addend = ((50F / byte.MaxValue) + (50F / byte.MaxValue)) * left; float expected = MathF.FusedMultiplyAdd(left, right, addend); - Assert.Equal(Vector128.Create(expected), Vector128_.FusedMultiplyAdd(Vector128.Create(left), Vector128.Create(right), Vector128.Create(addend))); - Assert.Equal(Vector256.Create(expected), Vector256_.FusedMultiplyAdd(Vector256.Create(left), Vector256.Create(right), Vector256.Create(addend))); - Assert.Equal(Vector512.Create(expected), Vector512_.FusedMultiplyAdd(Vector512.Create(left), Vector512.Create(right), Vector512.Create(addend))); + Assert.Equal(Vector128.Create(expected), Vector128.FusedMultiplyAdd(Vector128.Create(left), Vector128.Create(right), Vector128.Create(addend))); + Assert.Equal(Vector256.Create(expected), Vector256.FusedMultiplyAdd(Vector256.Create(left), Vector256.Create(right), Vector256.Create(addend))); + Assert.Equal(Vector512.Create(expected), Vector512.FusedMultiplyAdd(Vector512.Create(left), Vector512.Create(right), Vector512.Create(addend))); } private static void TestImpl_BulkConvertByteToNormalizedFloat( diff --git a/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs b/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs deleted file mode 100644 index 3437e800b..000000000 --- a/tests/ImageSharp.Tests/Common/TensorPrimitivesTests.cs +++ /dev/null @@ -1,580 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Common.Helpers; -using SixLabors.ImageSharp.Tests.TestUtilities; - -namespace SixLabors.ImageSharp.Tests.Common; - -public class TensorPrimitivesTests -{ - private static readonly int[] SpanLengthValues = - [ - 0, - 1, - 3, - 4, - 5, - 7, - 8, - 9, - 15, - 16, - 17, - 31, - 32, - 33, - 63, - 64, - 65, - 127, - 128, - 129, - 2048 - ]; - - /// - /// Gets lengths that exercise scalar execution, every SIMD width, overlapping tails, and the unrolled loop. - /// - public static TheoryData SpanLengths => new(SpanLengthValues); - - /// - /// Verifies every compatibility operation while forcing the supported SIMD feature tiers in isolated processes. - /// - [Fact] - public void OperationsMatchScalarFormulasAcrossHardwareIntrinsicFeatures() - => FeatureTestRunner.RunWithHwIntrinsicsFeature( - RunOperationsAcrossHardwareIntrinsicFeatures, - HwIntrinsics.AllowAll - | HwIntrinsics.DisableAVX512F - | HwIntrinsics.DisableAVX - | HwIntrinsics.DisableArm64Sve - | HwIntrinsics.DisableHWIntrinsic); - - /// - /// Verifies that span-to-span operations require equal input lengths before accessing either input. - /// - [Fact] - public void AddSpanSpanRejectsMismatchedInputLengths() - { - ArgumentException exception = Assert.Throws( - () => TensorPrimitives_.Add(new int[4], new int[3], new int[4])); - - Assert.Null(exception.ParamName); - } - - /// - /// Verifies that every operation rejects a destination that cannot hold all input elements. - /// - [Fact] - public void OperationsRejectShortDestinations() - { - int[] integers = new int[4]; - float[] singles = new float[4]; - - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Add(integers, integers, new int[3])).ParamName); - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Add(integers, 1, new int[3])).ParamName); - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Multiply(singles, 2F, new float[3])).ParamName); - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Divide(singles, 2F, new float[3])).ParamName); - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Max(singles, 2F, new float[3])).ParamName); - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Clamp(singles, 0F, 1F, new float[3])).ParamName); - Assert.Equal("destination", Assert.Throws(() => TensorPrimitives_.Negate(singles, new float[3])).ParamName); - } - - /// - /// Verifies that every operation rejects shifted input and destination overlap. - /// - [Fact] - public void OperationsRejectShiftedOverlap() - { - int[] integers = new int[5]; - int[] separateIntegers = new int[4]; - float[] singles = new float[5]; - - // Both inputs need independent validation because either one may alias a shifted destination. - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Add(integers.AsSpan(0, 4), separateIntegers, integers.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Add(separateIntegers, integers.AsSpan(0, 4), integers.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Add(integers.AsSpan(0, 4), 1, integers.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Multiply(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Divide(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Max(singles.AsSpan(0, 4), 2F, singles.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Clamp(singles.AsSpan(0, 4), 0F, 1F, singles.AsSpan(1, 4))).ParamName); - - Assert.Equal( - "destination", - Assert.Throws( - () => TensorPrimitives_.Negate(singles.AsSpan(0, 4), singles.AsSpan(1, 4))).ParamName); - } - - /// - /// Runs the TensorPrimitives compatibility assertions inside a process configured for one hardware-intrinsic tier. - /// - private static void RunOperationsAcrossHardwareIntrinsicFeatures() - { - TensorPrimitivesTests tests = new(); - - // Reuse the focused assertions so the remote feature matrix cannot drift from the normal test coverage. - foreach (int length in SpanLengthValues) - { - tests.AddByteMatchesScalarFormula(length); - tests.AddUInt32MatchesScalarFormula(length); - tests.AddScalarInt32MatchesScalarFormula(length); - tests.NegateSingleMatchesScalarFormula(length); - tests.NegateDoubleMatchesScalarFormula(length); - tests.ClampInt32MatchesScalarFormula(length); - tests.ClampSingleMatchesRuntimeFormula(length); - tests.DivideSingleMatchesScalarFormula(length); - tests.MaxSingleMatchesRuntimeFormula(length); - tests.MultiplySingleMatchesScalarFormula(length); - tests.NormalizeMatchesScalarFormula(length); - } - - tests.ClampSinglePreservesRuntimeSpecialValueSemantics(); - tests.ClampDoublePreservesRuntimeSpecialValueSemantics(); - } - - /// - /// Verifies that byte addition wraps modulo 256 and supports either input as the in-place destination. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void AddByteMatchesScalarFormula(int length) - { - byte[] x = new byte[length]; - byte[] y = new byte[length]; - byte[] expected = new byte[length]; - - for (int i = 0; i < length; i++) - { - x[i] = (byte)((i * 23) + 197); - y[i] = (byte)((i * 41) + 113); - expected[i] = unchecked((byte)(x[i] + y[i])); - } - - byte[] destination = new byte[length]; - TensorPrimitives_.Add(x, y, destination); - Assert.Equal(expected, destination); - - byte[] xInPlace = (byte[])x.Clone(); - TensorPrimitives_.Add(xInPlace, y, xInPlace); - Assert.Equal(expected, xInPlace); - - byte[] yInPlace = (byte[])y.Clone(); - TensorPrimitives_.Add(x, yInPlace, yInPlace); - Assert.Equal(expected, yInPlace); - } - - /// - /// Verifies that unsigned integer addition preserves unchecked histogram accumulation semantics. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void AddUInt32MatchesScalarFormula(int length) - { - uint[] x = new uint[length]; - uint[] y = new uint[length]; - uint[] expected = new uint[length]; - - for (int i = 0; i < length; i++) - { - x[i] = ((uint)i * 1_234_567U) + 0xF0000000U; - y[i] = ((uint)i * 7_654_321U) + 0x30000000U; - expected[i] = unchecked(x[i] + y[i]); - } - - TensorPrimitives_.Add(x, y, x); - Assert.Equal(expected, x); - } - - /// - /// Verifies that scalar integer addition produces identical results for separate and in-place destinations. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void AddScalarInt32MatchesScalarFormula(int length) - { - int[] source = new int[length]; - int[] expected = new int[length]; - const int addend = 17; - - for (int i = 0; i < length; i++) - { - source[i] = (i * 37) - 200; - expected[i] = source[i] + addend; - } - - int[] destination = new int[length]; - TensorPrimitives_.Add(source, addend, destination); - Assert.Equal(expected, destination); - - int[] inPlace = (int[])source.Clone(); - TensorPrimitives_.Add(inPlace, addend, inPlace); - Assert.Equal(expected, inPlace); - } - - /// - /// Verifies that floating-point negation preserves the scalar operator's exact bit-level behavior. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void NegateSingleMatchesScalarFormula(int length) - { - float[] values = - [ - float.NaN, - -0F, - 0F, - -1F, - 1F, - float.NegativeInfinity, - float.PositiveInfinity - ]; - - float[] source = new float[length]; - float[] expected = new float[length]; - - for (int i = 0; i < source.Length; i++) - { - source[i] = values[i % values.Length]; - expected[i] = -source[i]; - } - - float[] destination = new float[length]; - TensorPrimitives_.Negate(source, destination); - AssertSingleBitsEqual(expected, destination); - - TensorPrimitives_.Negate(source, source); - AssertSingleBitsEqual(expected, source); - } - - /// - /// Verifies that double-precision negation preserves the scalar operator's exact bit-level behavior. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void NegateDoubleMatchesScalarFormula(int length) - { - double[] values = - [ - double.NaN, - -0D, - 0D, - -1D, - 1D, - double.NegativeInfinity, - double.PositiveInfinity - ]; - - double[] source = new double[length]; - double[] expected = new double[length]; - - for (int i = 0; i < source.Length; i++) - { - source[i] = values[i % values.Length]; - expected[i] = -source[i]; - } - - double[] destination = new double[length]; - TensorPrimitives_.Negate(source, destination); - AssertDoubleBitsEqual(expected, destination); - - TensorPrimitives_.Negate(source, source); - AssertDoubleBitsEqual(expected, source); - } - - /// - /// Verifies that integer clamping produces identical results for separate and in-place destinations. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void ClampInt32MatchesScalarFormula(int length) - { - int[] source = new int[length]; - int[] expected = new int[length]; - - for (int i = 0; i < source.Length; i++) - { - source[i] = ((i * 37) % 401) - 200; - expected[i] = Math.Clamp(source[i], -73, 91); - } - - int[] destination = new int[length]; - TensorPrimitives_.Clamp(source, -73, 91, destination); - Assert.Equal(expected, destination); - - int[] inPlace = (int[])source.Clone(); - TensorPrimitives_.Clamp(inPlace, -73, 91, inPlace); - Assert.Equal(expected, inPlace); - } - - /// - /// Verifies that floating-point clamping matches the runtime tensor formula for special values and unordered bounds. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void ClampSingleMatchesRuntimeFormula(int length) - { - float[] values = - [ - float.NaN, - -0F, - 0F, - -1F, - 1F, - float.NegativeInfinity, - float.PositiveInfinity - ]; - - float[] source = new float[length]; - float[] expected = new float[length]; - - for (int i = 0; i < source.Length; i++) - { - source[i] = values[i % values.Length]; - - // Runtime main follows Min(Max(x, min), max) for vectorizable types, including unordered bounds. - expected[i] = float.Min(float.Max(source[i], 2F), -2F); - } - - TensorPrimitives_.Clamp(source, 2F, -2F, source); - AssertSingleBitsEqual(expected, source); - } - - /// - /// Verifies that single-precision clamping preserves the runtime's signed-zero and NaN behavior. - /// - [Fact] - public void ClampSinglePreservesRuntimeSpecialValueSemantics() - { - float[] values = - [ - float.NaN, - float.NegativeInfinity, - -0F, - 0F, - float.PositiveInfinity - ]; - - float[] actual = new float[129]; - float[] expected = new float[actual.Length]; - - for (int i = 0; i < actual.Length; i++) - { - actual[i] = values[i % values.Length]; - expected[i] = float.Min(float.Max(actual[i], -0F), 0F); - } - - TensorPrimitives_.Clamp(actual, -0F, 0F, actual); - AssertSingleBitsEqual(expected, actual); - } - - /// - /// Verifies that double-precision clamping preserves the runtime's signed-zero and NaN behavior. - /// - [Fact] - public void ClampDoublePreservesRuntimeSpecialValueSemantics() - { - double[] values = - [ - double.NaN, - double.NegativeInfinity, - -0D, - 0D, - double.PositiveInfinity - ]; - - double[] actual = new double[65]; - double[] expected = new double[actual.Length]; - - for (int i = 0; i < actual.Length; i++) - { - actual[i] = values[i % values.Length]; - expected[i] = double.Min(double.Max(actual[i], -0D), 0D); - } - - TensorPrimitives_.Clamp(actual, -0D, 0D, actual); - AssertDoubleBitsEqual(expected, actual); - } - - /// - /// Verifies that division produces identical results for separate and in-place destinations. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void DivideSingleMatchesScalarFormula(int length) - { - float[] source = new float[length]; - float[] expected = new float[length]; - - for (int i = 0; i < source.Length; i++) - { - source[i] = (i - 65.25F) * 1.75F; - expected[i] = source[i] / 3.25F; - } - - float[] destination = new float[length]; - TensorPrimitives_.Divide(source, 3.25F, destination); - AssertSingleBitsEqual(expected, destination); - - float[] inPlace = (float[])source.Clone(); - TensorPrimitives_.Divide(inPlace, 3.25F, inPlace); - AssertSingleBitsEqual(expected, inPlace); - } - - /// - /// Verifies that maximum selection preserves the runtime's NaN and signed-zero semantics. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void MaxSingleMatchesRuntimeFormula(int length) - { - float[] values = - [ - float.NaN, - float.NegativeInfinity, - -1F, - -0F, - 0F, - 1F, - float.PositiveInfinity - ]; - - float[] actual = new float[length]; - float[] expected = new float[length]; - - for (int i = 0; i < length; i++) - { - actual[i] = values[i % values.Length]; - expected[i] = float.Max(actual[i], -0F); - } - - TensorPrimitives_.Max(actual, -0F, actual); - AssertSingleBitsEqual(expected, actual); - } - - /// - /// Verifies that multiplication produces identical results for separate and in-place destinations. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void MultiplySingleMatchesScalarFormula(int length) - { - float[] source = new float[length]; - float[] expected = new float[length]; - - for (int i = 0; i < source.Length; i++) - { - source[i] = (i - 65.25F) * 1.75F; - expected[i] = source[i] * 0.375F; - } - - float[] destination = new float[length]; - TensorPrimitives_.Multiply(source, 0.375F, destination); - AssertSingleBitsEqual(expected, destination); - - TensorPrimitives_.Multiply(source, 0.375F, source); - AssertSingleBitsEqual(expected, source); - } - - /// - /// Verifies that the normalization compatibility call preserves its element-wise division contract. - /// - /// The input length. - [Theory] - [MemberData(nameof(SpanLengths))] - public void NormalizeMatchesScalarFormula(int length) - { - float[] actual = new float[length]; - float[] expected = new float[length]; - - for (int i = 0; i < actual.Length; i++) - { - actual[i] = (i + 1) * 0.125F; - expected[i] = actual[i] / 7.5F; - } - - Numerics.Normalize(actual, 7.5F); - AssertSingleBitsEqual(expected, actual); - } - - /// - /// Compares floating-point results while preserving signed-zero behavior. - /// - /// The expected values. - /// The actual values. - private static void AssertSingleBitsEqual(ReadOnlySpan expected, ReadOnlySpan actual) - { - Assert.Equal(expected.Length, actual.Length); - - for (int i = 0; i < expected.Length; i++) - { - if (float.IsNaN(expected[i])) - { - Assert.True(float.IsNaN(actual[i])); - } - else - { - Assert.Equal(BitConverter.SingleToInt32Bits(expected[i]), BitConverter.SingleToInt32Bits(actual[i])); - } - } - } - - /// - /// Compares double-precision results while preserving signed-zero behavior. - /// - /// The expected values. - /// The actual values. - private static void AssertDoubleBitsEqual(ReadOnlySpan expected, ReadOnlySpan actual) - { - Assert.Equal(expected.Length, actual.Length); - - for (int i = 0; i < expected.Length; i++) - { - if (double.IsNaN(expected[i])) - { - Assert.True(double.IsNaN(actual[i])); - } - else - { - Assert.Equal(BitConverter.DoubleToInt64Bits(expected[i]), BitConverter.DoubleToInt64Bits(actual[i])); - } - } - } -}