diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs b/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs index d4ab8c6183..c856267db2 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs @@ -1,12 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Buffers.Binary; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using static SixLabors.ImageSharp.SimdUtils; - // The JIT can detect and optimize rotation idioms ROTL (Rotate Left) // and ROTR (Rotate Right) emitting efficient CPU instructions: // https://github.com/dotnet/coreclr/pull/1830 @@ -20,7 +14,7 @@ internal interface IComponentShuffle { /// /// Shuffles then slices 8-bit integers in - /// using the control and store the results in . + /// using a byte control and store the results in . /// If successful, this method will reduce the length of length /// by the shuffle amount. /// @@ -40,168 +34,3 @@ internal interface IComponentShuffle /// void Shuffle(ReadOnlySpan source, Span destination); } - -/// -internal interface IShuffle4 : IComponentShuffle -{ -} - -internal readonly struct DefaultShuffle4(byte control) : IShuffle4 -{ - public byte Control { get; } = control; - - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, this.Control); - - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(destination); - - SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0); - - for (nuint i = 0; i < (uint)source.Length; i += 4) - { - Unsafe.Add(ref dBase, i + 0) = Unsafe.Add(ref sBase, p0 + i); - Unsafe.Add(ref dBase, i + 1) = Unsafe.Add(ref sBase, p1 + i); - Unsafe.Add(ref dBase, i + 2) = Unsafe.Add(ref sBase, p2 + i); - Unsafe.Add(ref dBase, i + 3) = Unsafe.Add(ref sBase, p3 + i); - } - } -} - -internal readonly struct WXYZShuffle4 : IShuffle4 -{ - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle2103); - - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // ROTL(8, packed) = [Z Y X W] - Unsafe.Add(ref dBase, i) = (packed << 8) | (packed >> 24); - } - } -} - -internal readonly struct WZYXShuffle4 : IShuffle4 -{ - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0123); - - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // REVERSE(packedArgb) = [X Y Z W] - Unsafe.Add(ref dBase, i) = BinaryPrimitives.ReverseEndianness(packed); - } - } -} - -internal readonly struct YZWXShuffle4 : IShuffle4 -{ - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0321); - - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // ROTR(8, packedArgb) = [Y Z W X] - Unsafe.Add(ref dBase, i) = BitOperations.RotateRight(packed, 8); - } - } -} - -internal readonly struct ZYXWShuffle4 : IShuffle4 -{ - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3012); - - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // tmp1 = [W 0 Y 0] - // tmp2 = [0 Z 0 X] - // tmp3=ROTL(16, tmp2) = [0 X 0 Z] - // tmp1 + tmp3 = [W X Y Z] - uint tmp1 = packed & 0xFF00FF00; - uint tmp2 = packed & 0x00FF00FF; - uint tmp3 = BitOperations.RotateLeft(tmp2, 16); - - Unsafe.Add(ref dBase, i) = tmp1 + tmp3; - } - } -} - -internal readonly struct XWZYShuffle4 : IShuffle4 -{ - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle1230); - - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // tmp1 = [0 Z 0 X] - // tmp2 = [W 0 Y 0] - // tmp3=ROTL(16, tmp2) = [Y 0 W 0] - // tmp1 + tmp3 = [Y Z W X] - uint tmp1 = packed & 0x00FF00FF; - uint tmp2 = packed & 0xFF00FF00; - uint tmp3 = BitOperations.RotateLeft(tmp2, 16); - - Unsafe.Add(ref dBase, i) = tmp1 + tmp3; - } - } -} diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs index 255448d619..0f282c7f9a 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using static SixLabors.ImageSharp.SimdUtils; @@ -12,22 +13,21 @@ internal interface IPad3Shuffle4 : IComponentShuffle { } -internal readonly struct DefaultPad3Shuffle4 : IPad3Shuffle4 +internal readonly struct DefaultPad3Shuffle4([ConstantExpected] byte control) : IPad3Shuffle4 { - public DefaultPad3Shuffle4(byte control) - => this.Control = control; - - public byte Control { get; } + public byte Control { get; } = control; [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span dest) - => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref dest, this.Control); + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) +#pragma warning disable CA1857 // A constant is expected for the parameter + => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, this.Control); +#pragma warning restore CA1857 // A constant is expected for the parameter [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span dest) + public void Shuffle(ReadOnlySpan source, Span destination) { ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(dest); + ref byte dBase = ref MemoryMarshal.GetReference(destination); SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0); @@ -51,14 +51,14 @@ internal readonly struct DefaultPad3Shuffle4 : IPad3Shuffle4 internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4 { [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span dest) - => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref dest, SimdUtils.Shuffle.MMShuffle3210); + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210); [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span dest) + public void Shuffle(ReadOnlySpan source, Span destination) { ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(dest); + ref byte dBase = ref MemoryMarshal.GetReference(destination); ref byte sEnd = ref Unsafe.Add(ref sBase, (uint)source.Length); ref byte sLoopEnd = ref Unsafe.Subtract(ref sEnd, 4); diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs index 89faca2437..3c0973ad69 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using static SixLabors.ImageSharp.SimdUtils; @@ -12,22 +13,21 @@ internal interface IShuffle3 : IComponentShuffle { } -internal readonly struct DefaultShuffle3 : IShuffle3 +internal readonly struct DefaultShuffle3([ConstantExpected] byte control) : IShuffle3 { - public DefaultShuffle3(byte control) - => this.Control = control; - - public byte Control { get; } + public byte Control { get; } = control; [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span dest) - => HwIntrinsics.Shuffle3Reduce(ref source, ref dest, this.Control); + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) +#pragma warning disable CA1857 // A constant is expected for the parameter + => HwIntrinsics.Shuffle3Reduce(ref source, ref destination, this.Control); +#pragma warning restore CA1857 // A constant is expected for the parameter [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span dest) + public void Shuffle(ReadOnlySpan source, Span destination) { ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(dest); + ref byte dBase = ref MemoryMarshal.GetReference(destination); SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0); diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs new file mode 100644 index 0000000000..d5c6df2c8b --- /dev/null +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs @@ -0,0 +1,178 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers.Binary; +using System.Diagnostics.CodeAnalysis; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using static SixLabors.ImageSharp.SimdUtils; + +namespace SixLabors.ImageSharp; + +/// +internal interface IShuffle4 : IComponentShuffle +{ +} + +internal readonly struct DefaultShuffle4([ConstantExpected] byte control) : IShuffle4 +{ + public byte Control { get; } = control; + + [MethodImpl(InliningOptions.ShortMethod)] + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) +#pragma warning disable CA1857 // A constant is expected for the parameter + => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, this.Control); +#pragma warning restore CA1857 // A constant is expected for the parameter + + [MethodImpl(InliningOptions.ShortMethod)] + public void Shuffle(ReadOnlySpan source, Span destination) + { + ref byte sBase = ref MemoryMarshal.GetReference(source); + ref byte dBase = ref MemoryMarshal.GetReference(destination); + + SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0); + + for (nuint i = 0; i < (uint)source.Length; i += 4) + { + Unsafe.Add(ref dBase, i + 0) = Unsafe.Add(ref sBase, p0 + i); + Unsafe.Add(ref dBase, i + 1) = Unsafe.Add(ref sBase, p1 + i); + Unsafe.Add(ref dBase, i + 2) = Unsafe.Add(ref sBase, p2 + i); + Unsafe.Add(ref dBase, i + 3) = Unsafe.Add(ref sBase, p3 + i); + } + } +} + +internal readonly struct WXYZShuffle4 : IShuffle4 +{ + [MethodImpl(InliningOptions.ShortMethod)] + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle2103); + + [MethodImpl(InliningOptions.ShortMethod)] + public void Shuffle(ReadOnlySpan source, Span destination) + { + ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); + ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + uint n = (uint)source.Length / 4; + + for (nuint i = 0; i < n; i++) + { + uint packed = Unsafe.Add(ref sBase, i); + + // packed = [W Z Y X] + // ROTL(8, packed) = [Z Y X W] + Unsafe.Add(ref dBase, i) = (packed << 8) | (packed >> 24); + } + } +} + +internal readonly struct WZYXShuffle4 : IShuffle4 +{ + [MethodImpl(InliningOptions.ShortMethod)] + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0123); + + [MethodImpl(InliningOptions.ShortMethod)] + public void Shuffle(ReadOnlySpan source, Span destination) + { + ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); + ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + uint n = (uint)source.Length / 4; + + for (nuint i = 0; i < n; i++) + { + uint packed = Unsafe.Add(ref sBase, i); + + // packed = [W Z Y X] + // REVERSE(packedArgb) = [X Y Z W] + Unsafe.Add(ref dBase, i) = BinaryPrimitives.ReverseEndianness(packed); + } + } +} + +internal readonly struct YZWXShuffle4 : IShuffle4 +{ + [MethodImpl(InliningOptions.ShortMethod)] + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0321); + + [MethodImpl(InliningOptions.ShortMethod)] + public void Shuffle(ReadOnlySpan source, Span destination) + { + ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); + ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + uint n = (uint)source.Length / 4; + + for (nuint i = 0; i < n; i++) + { + uint packed = Unsafe.Add(ref sBase, i); + + // packed = [W Z Y X] + // ROTR(8, packedArgb) = [Y Z W X] + Unsafe.Add(ref dBase, i) = BitOperations.RotateRight(packed, 8); + } + } +} + +internal readonly struct ZYXWShuffle4 : IShuffle4 +{ + [MethodImpl(InliningOptions.ShortMethod)] + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3012); + + [MethodImpl(InliningOptions.ShortMethod)] + public void Shuffle(ReadOnlySpan source, Span destination) + { + ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); + ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + uint n = (uint)source.Length / 4; + + for (nuint i = 0; i < n; i++) + { + uint packed = Unsafe.Add(ref sBase, i); + + // packed = [W Z Y X] + // tmp1 = [W 0 Y 0] + // tmp2 = [0 Z 0 X] + // tmp3=ROTL(16, tmp2) = [0 X 0 Z] + // tmp1 + tmp3 = [W X Y Z] + uint tmp1 = packed & 0xFF00FF00; + uint tmp2 = packed & 0x00FF00FF; + uint tmp3 = BitOperations.RotateLeft(tmp2, 16); + + Unsafe.Add(ref dBase, i) = tmp1 + tmp3; + } + } +} + +internal readonly struct XWZYShuffle4 : IShuffle4 +{ + [MethodImpl(InliningOptions.ShortMethod)] + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle1230); + + [MethodImpl(InliningOptions.ShortMethod)] + public void Shuffle(ReadOnlySpan source, Span destination) + { + ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); + ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + uint n = (uint)source.Length / 4; + + for (nuint i = 0; i < n; i++) + { + uint packed = Unsafe.Add(ref sBase, i); + + // packed = [W Z Y X] + // tmp1 = [0 Z 0 X] + // tmp2 = [W 0 Y 0] + // tmp3=ROTL(16, tmp2) = [Y 0 W 0] + // tmp1 + tmp3 = [Y Z W X] + uint tmp1 = packed & 0x00FF00FF; + uint tmp2 = packed & 0xFF00FF00; + uint tmp3 = BitOperations.RotateLeft(tmp2, 16); + + Unsafe.Add(ref dBase, i) = tmp1 + tmp3; + } + } +} diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs index 30fda7a8e1..3e7e440664 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using static SixLabors.ImageSharp.SimdUtils; @@ -12,26 +13,25 @@ internal interface IShuffle4Slice3 : IComponentShuffle { } -internal readonly struct DefaultShuffle4Slice3 : IShuffle4Slice3 +internal readonly struct DefaultShuffle4Slice3([ConstantExpected] byte control) : IShuffle4Slice3 { - public DefaultShuffle4Slice3(byte control) - => this.Control = control; - - public byte Control { get; } + public byte Control { get; } = control; [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span dest) - => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref dest, this.Control); + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) +#pragma warning disable CA1857 // A constant is expected for the parameter + => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, this.Control); +#pragma warning restore CA1857 // A constant is expected for the parameter [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span dest) + public void Shuffle(ReadOnlySpan source, Span destination) { ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(dest); + ref byte dBase = ref MemoryMarshal.GetReference(destination); SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0); - for (nuint i = 0, j = 0; i < (uint)dest.Length; i += 3, j += 4) + for (nuint i = 0, j = 0; i < (uint)destination.Length; i += 3, j += 4) { Unsafe.Add(ref dBase, i + 0) = Unsafe.Add(ref sBase, p0 + j); Unsafe.Add(ref dBase, i + 1) = Unsafe.Add(ref sBase, p1 + j); @@ -43,14 +43,14 @@ internal readonly struct DefaultShuffle4Slice3 : IShuffle4Slice3 internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3 { [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span dest) - => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref dest, SimdUtils.Shuffle.MMShuffle3210); + public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) + => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210); [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span dest) + public void Shuffle(ReadOnlySpan source, Span destination) { ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref Byte3 dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(dest)); + ref Byte3 dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); nint n = (nint)(uint)source.Length / 4; nint m = Numerics.Modulo4(n); diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs b/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs index 4b9a90a952..14d2f1c259 100644 --- a/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs +++ b/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs @@ -63,7 +63,9 @@ internal static partial class SimdUtils ref Span destination, [ConstantExpected] byte control) { - if (Vector512.IsHardwareAccelerated || Vector256.IsHardwareAccelerated || Vector128.IsHardwareAccelerated) + if ((Vector512.IsHardwareAccelerated && Vector512Utilities.SupportsShuffleFloat) || + (Vector256.IsHardwareAccelerated && Vector256Utilities.SupportsShuffleFloat) || + (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleFloat)) { int remainder = 0; if (Vector512.IsHardwareAccelerated) @@ -105,9 +107,11 @@ internal static partial class SimdUtils public static void Shuffle4Reduce( ref ReadOnlySpan source, ref Span destination, - byte control) + [ConstantExpected] byte control) { - if (Vector512.IsHardwareAccelerated || Vector256.IsHardwareAccelerated || Vector128.IsHardwareAccelerated) + if ((Vector512.IsHardwareAccelerated && Vector512Utilities.SupportsShuffleByte) || + (Vector256.IsHardwareAccelerated && Vector256Utilities.SupportsShuffleByte) || + (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte)) { int remainder = 0; if (Vector512.IsHardwareAccelerated) @@ -139,7 +143,7 @@ internal static partial class SimdUtils } /// - /// Shuffles 8-bit integer triplets within 128-bit lanes in + /// Shuffles 8-bit integer triplets in /// using the control and store the results in . /// /// The source span of bytes. @@ -149,9 +153,9 @@ internal static partial class SimdUtils public static void Shuffle3Reduce( ref ReadOnlySpan source, ref Span destination, - byte control) + [ConstantExpected] byte control) { - if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsRightShift) + if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte && Vector128Utilities.SupportsRightAlign) { int remainder = source.Length % (Vector128.Count * 3); @@ -171,67 +175,67 @@ internal static partial class SimdUtils } /// - /// Pads then shuffles 8-bit integers within 128-bit lanes in - /// using the control and store the results in . + /// Pads then shuffles 8-bit integers in + /// using the control and store the results in . /// /// The source span of bytes. - /// The destination span of bytes. + /// The destination span of bytes. /// The byte control. [MethodImpl(InliningOptions.ShortMethod)] public static void Pad3Shuffle4Reduce( ref ReadOnlySpan source, - ref Span dest, - byte control) + ref Span destination, + [ConstantExpected] byte control) { - if (Ssse3.IsSupported) + if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte && Vector128Utilities.SupportsShiftByte) { int remainder = source.Length % (Vector128.Count * 3); int sourceCount = source.Length - remainder; - int destCount = (int)((uint)sourceCount * 4 / 3); + int destinationCount = (int)((uint)sourceCount * 4 / 3); if (sourceCount > 0) { Pad3Shuffle4( source[..sourceCount], - dest[..destCount], + destination[..destinationCount], control); source = source[sourceCount..]; - dest = dest[destCount..]; + destination = destination[destinationCount..]; } } } /// - /// Shuffles then slices 8-bit integers within 128-bit lanes in - /// using the control and store the results in . + /// Shuffles then slices 8-bit integers in + /// using the control and store the results in . /// /// The source span of bytes. - /// The destination span of bytes. + /// The destination span of bytes. /// The byte control. [MethodImpl(InliningOptions.ShortMethod)] public static void Shuffle4Slice3Reduce( ref ReadOnlySpan source, - ref Span dest, - byte control) + ref Span destination, + [ConstantExpected] byte control) { - if (Ssse3.IsSupported) + if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte && Vector128Utilities.SupportsShiftByte) { int remainder = source.Length & ((Vector128.Count * 4) - 1); // bit-hack for modulo int sourceCount = source.Length - remainder; - int destCount = (int)((uint)sourceCount * 3 / 4); + int destinationCount = (int)((uint)sourceCount * 3 / 4); if (sourceCount > 0) { Shuffle4Slice3( source[..sourceCount], - dest[..destCount], + destination[..destinationCount], control); source = source[sourceCount..]; - dest = dest[destCount..]; + destination = destination[destinationCount..]; } } } @@ -242,12 +246,8 @@ internal static partial class SimdUtils Span destination, [ConstantExpected] byte control) { - if (Vector512.IsHardwareAccelerated) + if (Vector512.IsHardwareAccelerated && Vector512Utilities.SupportsShuffleFloat) { - Span temp = stackalloc int[Vector512.Count]; - Shuffle.MMShuffleSpan(ref temp, control); - Vector512 mask = Unsafe.As>(ref MemoryMarshal.GetReference(temp)); - ref Vector512 sourceBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); ref Vector512 destinationBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); @@ -260,26 +260,22 @@ internal static partial class SimdUtils ref Vector512 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector512 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector512.Shuffle(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector512.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector512.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector512.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector512Utilities.Shuffle(vs0, control); + Unsafe.Add(ref vd0, (nuint)1) = Vector512Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)1), control); + Unsafe.Add(ref vd0, (nuint)2) = Vector512Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)2), control); + Unsafe.Add(ref vd0, (nuint)3) = Vector512Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)3), control); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector512.Shuffle(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector512Utilities.Shuffle(Unsafe.Add(ref sourceBase, i), control); } } } - else if (Vector256.IsHardwareAccelerated) + else if (Vector256.IsHardwareAccelerated && Vector256Utilities.SupportsShuffleFloat) { - Span temp = stackalloc int[Vector256.Count]; - Shuffle.MMShuffleSpan(ref temp, control); - Vector256 mask = Unsafe.As>(ref MemoryMarshal.GetReference(temp)); - ref Vector256 sourceBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); ref Vector256 destinationBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); @@ -292,26 +288,22 @@ internal static partial class SimdUtils ref Vector256 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector256 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector256.Shuffle(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector256.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector256.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector256.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector256Utilities.Shuffle(vs0, control); + Unsafe.Add(ref vd0, (nuint)1) = Vector256Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)1), control); + Unsafe.Add(ref vd0, (nuint)2) = Vector256Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)2), control); + Unsafe.Add(ref vd0, (nuint)3) = Vector256Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)3), control); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector256.Shuffle(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector256Utilities.Shuffle(Unsafe.Add(ref sourceBase, i), control); } } } - else if (Vector128.IsHardwareAccelerated) + else if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleFloat) { - Span temp = stackalloc int[Vector128.Count]; - Shuffle.MMShuffleSpan(ref temp, control); - Vector128 mask = Unsafe.As>(ref MemoryMarshal.GetReference(temp)); - ref Vector128 sourceBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); ref Vector128 destinationBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); @@ -324,17 +316,17 @@ internal static partial class SimdUtils ref Vector128 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector128 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector128.Shuffle(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector128.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector128.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector128.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector128Utilities.Shuffle(vs0, control); + Unsafe.Add(ref vd0, (nuint)1) = Vector128Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)1), control); + Unsafe.Add(ref vd0, (nuint)2) = Vector128Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)2), control); + Unsafe.Add(ref vd0, (nuint)3) = Vector128Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)3), control); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector128.Shuffle(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector128Utilities.Shuffle(Unsafe.Add(ref sourceBase, i), control); } } } @@ -344,9 +336,9 @@ internal static partial class SimdUtils private static void Shuffle4( ReadOnlySpan source, Span destination, - byte control) + [ConstantExpected] byte control) { - if (Vector512.IsHardwareAccelerated) + if (Vector512.IsHardwareAccelerated && Vector512Utilities.SupportsShuffleByte) { Span temp = stackalloc byte[Vector512.Count]; Shuffle.MMShuffleSpan(ref temp, control); @@ -364,21 +356,21 @@ internal static partial class SimdUtils ref Vector512 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector512 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector512.Shuffle(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector512.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector512.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector512.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector512Utilities.Shuffle(vs0, mask); + Unsafe.Add(ref vd0, (nuint)1) = Vector512Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); + Unsafe.Add(ref vd0, (nuint)2) = Vector512Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); + Unsafe.Add(ref vd0, (nuint)3) = Vector512Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector512.Shuffle(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector512Utilities.Shuffle(Unsafe.Add(ref sourceBase, i), mask); } } } - else if (Vector256.IsHardwareAccelerated) + else if (Vector256.IsHardwareAccelerated && Vector256Utilities.SupportsShuffleByte) { Span temp = stackalloc byte[Vector256.Count]; Shuffle.MMShuffleSpan(ref temp, control); @@ -396,21 +388,21 @@ internal static partial class SimdUtils ref Vector256 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector256 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector256.Shuffle(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector256.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector256.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector256.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector256Utilities.Shuffle(vs0, mask); + Unsafe.Add(ref vd0, (nuint)1) = Vector256Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); + Unsafe.Add(ref vd0, (nuint)2) = Vector256Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); + Unsafe.Add(ref vd0, (nuint)3) = Vector256Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector256.Shuffle(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector256Utilities.Shuffle(Unsafe.Add(ref sourceBase, i), mask); } } } - else if (Vector128.IsHardwareAccelerated) + else if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte) { Span temp = stackalloc byte[Vector128.Count]; Shuffle.MMShuffleSpan(ref temp, control); @@ -428,17 +420,17 @@ internal static partial class SimdUtils ref Vector128 vs0 = ref Unsafe.Add(ref sourceBase, i); ref Vector128 vd0 = ref Unsafe.Add(ref destinationBase, i); - vd0 = Vector128.Shuffle(vs0, mask); - Unsafe.Add(ref vd0, (nuint)1) = Vector128.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); - Unsafe.Add(ref vd0, (nuint)2) = Vector128.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); - Unsafe.Add(ref vd0, (nuint)3) = Vector128.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); + vd0 = Vector128Utilities.Shuffle(vs0, mask); + Unsafe.Add(ref vd0, (nuint)1) = Vector128Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)1), mask); + Unsafe.Add(ref vd0, (nuint)2) = Vector128Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)2), mask); + Unsafe.Add(ref vd0, (nuint)3) = Vector128Utilities.Shuffle(Unsafe.Add(ref vs0, (nuint)3), mask); } if (m > 0) { for (nuint i = u; i < n; i++) { - Unsafe.Add(ref destinationBase, i) = Vector128.Shuffle(Unsafe.Add(ref sourceBase, i), mask); + Unsafe.Add(ref destinationBase, i) = Vector128Utilities.Shuffle(Unsafe.Add(ref sourceBase, i), mask); } } } @@ -448,9 +440,9 @@ internal static partial class SimdUtils private static void Shuffle3( ReadOnlySpan source, Span destination, - byte control) + [ConstantExpected] byte control) { - if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsRightShift) + if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte && Vector128Utilities.SupportsRightAlign) { Vector128 maskPad4Nx16 = ShuffleMaskPad4Nx16(); Vector128 maskSlice4Nx16 = ShuffleMaskSlice4Nx16(); @@ -477,15 +469,15 @@ internal static partial class SimdUtils v2 = Vector128Utilities.AlignRight(v2, v1, 8); v1 = Vector128Utilities.AlignRight(v1, v0, 12); - v0 = Vector128.Shuffle(Vector128.Shuffle(v0, maskPad4Nx16), mask); - v1 = Vector128.Shuffle(Vector128.Shuffle(v1, maskPad4Nx16), mask); - v2 = Vector128.Shuffle(Vector128.Shuffle(v2, maskPad4Nx16), mask); - v3 = Vector128.Shuffle(Vector128.Shuffle(v3, maskPad4Nx16), mask); + v0 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v0, maskPad4Nx16), mask); + v1 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v1, maskPad4Nx16), mask); + v2 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v2, maskPad4Nx16), mask); + v3 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v3, maskPad4Nx16), mask); - v0 = Vector128.Shuffle(v0, maskE); - v1 = Vector128.Shuffle(v1, maskSlice4Nx16); - v2 = Vector128.Shuffle(v2, maskE); - v3 = Vector128.Shuffle(v3, maskSlice4Nx16); + v0 = Vector128Utilities.Shuffle(v0, maskE); + v1 = Vector128Utilities.Shuffle(v1, maskSlice4Nx16); + v2 = Vector128Utilities.Shuffle(v2, maskE); + v3 = Vector128Utilities.Shuffle(v3, maskSlice4Nx16); v0 = Vector128Utilities.AlignRight(v1, v0, 4); v3 = Vector128Utilities.AlignRight(v3, v2, 12); @@ -508,21 +500,21 @@ internal static partial class SimdUtils private static void Pad3Shuffle4( ReadOnlySpan source, Span destination, - byte control) + [ConstantExpected] byte control) { - if (Ssse3.IsSupported) + if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte && Vector128Utilities.SupportsShiftByte) { - Vector128 vmask = ShuffleMaskPad4Nx16(); - Vector128 vfill = Vector128.Create(0xff000000ff000000ul).AsByte(); + Vector128 maskPad4Nx16 = ShuffleMaskPad4Nx16(); + Vector128 fill = Vector128.Create(0xff000000ff000000ul).AsByte(); - Span bytes = stackalloc byte[Vector128.Count]; - Shuffle.MMShuffleSpan(ref bytes, control); - Vector128 vshuffle = Unsafe.As>(ref MemoryMarshal.GetReference(bytes)); + Span temp = stackalloc byte[Vector128.Count]; + Shuffle.MMShuffleSpan(ref temp, control); + Vector128 mask = Unsafe.As>(ref MemoryMarshal.GetReference(temp)); ref Vector128 sourceBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); - ref Vector128 destBase = + ref Vector128 destinationBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); nuint n = source.Vector128Count(); @@ -532,17 +524,17 @@ internal static partial class SimdUtils ref Vector128 v0 = ref Unsafe.Add(ref sourceBase, i); Vector128 v1 = Unsafe.Add(ref v0, 1); Vector128 v2 = Unsafe.Add(ref v0, 2); - Vector128 v3 = Sse2.ShiftRightLogical128BitLane(v2, 4); + Vector128 v3 = Vector128Utilities.ShiftRightBytesInVector(v2, 4); - v2 = Ssse3.AlignRight(v2, v1, 8); - v1 = Ssse3.AlignRight(v1, v0, 12); + v2 = Vector128Utilities.AlignRight(v2, v1, 8); + v1 = Vector128Utilities.AlignRight(v1, v0, 12); - ref Vector128 vd = ref Unsafe.Add(ref destBase, j); + ref Vector128 vd = ref Unsafe.Add(ref destinationBase, j); - vd = Ssse3.Shuffle(Sse2.Or(Ssse3.Shuffle(v0, vmask), vfill), vshuffle); - Unsafe.Add(ref vd, 1) = Ssse3.Shuffle(Sse2.Or(Ssse3.Shuffle(v1, vmask), vfill), vshuffle); - Unsafe.Add(ref vd, 2) = Ssse3.Shuffle(Sse2.Or(Ssse3.Shuffle(v2, vmask), vfill), vshuffle); - Unsafe.Add(ref vd, 3) = Ssse3.Shuffle(Sse2.Or(Ssse3.Shuffle(v3, vmask), vfill), vshuffle); + vd = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v0, maskPad4Nx16) | fill, mask); + Unsafe.Add(ref vd, 1) = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v1, maskPad4Nx16) | fill, mask); + Unsafe.Add(ref vd, 2) = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v2, maskPad4Nx16) | fill, mask); + Unsafe.Add(ref vd, 3) = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v3, maskPad4Nx16) | fill, mask); } } } @@ -550,23 +542,23 @@ internal static partial class SimdUtils [MethodImpl(InliningOptions.ShortMethod)] private static void Shuffle4Slice3( ReadOnlySpan source, - Span dest, - byte control) + Span destination, + [ConstantExpected] byte control) { - if (Ssse3.IsSupported) + if (Vector128.IsHardwareAccelerated && Vector128Utilities.SupportsShuffleByte && Vector128Utilities.SupportsShiftByte) { - Vector128 vmasko = ShuffleMaskSlice4Nx16(); - Vector128 vmaske = Ssse3.AlignRight(vmasko, vmasko, 12); + Vector128 maskSlice4Nx16 = ShuffleMaskSlice4Nx16(); + Vector128 maskE = Ssse3.AlignRight(maskSlice4Nx16, maskSlice4Nx16, 12); - Span bytes = stackalloc byte[Vector128.Count]; - Shuffle.MMShuffleSpan(ref bytes, control); - Vector128 vshuffle = Unsafe.As>(ref MemoryMarshal.GetReference(bytes)); + Span temp = stackalloc byte[Vector128.Count]; + Shuffle.MMShuffleSpan(ref temp, control); + Vector128 mask = Unsafe.As>(ref MemoryMarshal.GetReference(temp)); ref Vector128 sourceBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); - ref Vector128 destBase = - ref Unsafe.As>(ref MemoryMarshal.GetReference(dest)); + ref Vector128 destinationBase = + ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); nuint n = source.Vector128Count(); @@ -579,20 +571,20 @@ internal static partial class SimdUtils Vector128 v2 = Unsafe.Add(ref vs, 2); Vector128 v3 = Unsafe.Add(ref vs, 3); - v0 = Ssse3.Shuffle(Ssse3.Shuffle(v0, vshuffle), vmaske); - v1 = Ssse3.Shuffle(Ssse3.Shuffle(v1, vshuffle), vmasko); - v2 = Ssse3.Shuffle(Ssse3.Shuffle(v2, vshuffle), vmaske); - v3 = Ssse3.Shuffle(Ssse3.Shuffle(v3, vshuffle), vmasko); + v0 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v0, mask), maskE); + v1 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v1, mask), maskSlice4Nx16); + v2 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v2, mask), maskE); + v3 = Vector128Utilities.Shuffle(Vector128Utilities.Shuffle(v3, mask), maskSlice4Nx16); - v0 = Ssse3.AlignRight(v1, v0, 4); - v3 = Ssse3.AlignRight(v3, v2, 12); + v0 = Vector128Utilities.AlignRight(v1, v0, 4); + v3 = Vector128Utilities.AlignRight(v3, v2, 12); - v1 = Sse2.ShiftLeftLogical128BitLane(v1, 4); - v2 = Sse2.ShiftRightLogical128BitLane(v2, 4); + v1 = Vector128Utilities.ShiftLeftBytesInVector(v1, 4); + v2 = Vector128Utilities.ShiftRightBytesInVector(v2, 4); - v1 = Ssse3.AlignRight(v2, v1, 8); + v1 = Vector128Utilities.AlignRight(v2, v1, 8); - ref Vector128 vd = ref Unsafe.Add(ref destBase, j); + ref Vector128 vd = ref Unsafe.Add(ref destinationBase, j); vd = v0; Unsafe.Add(ref vd, 1) = v1; diff --git a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs index 829362da88..a272e459cb 100644 --- a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs +++ b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs @@ -11,20 +11,95 @@ using System.Runtime.Intrinsics.X86; namespace SixLabors.ImageSharp.Common.Helpers; /// -/// Defines utility methods for that have not yet been normalized in the runtime. +/// Defines utility methods for that have either: +/// +/// Not yet been normalized in the runtime. +/// Produce codegen that is poorly optimized by the runtime. +/// /// Should only be used if the intrinsics are available. /// internal static class Vector128Utilities { /// - /// Gets a value indicating whether right shift operations are supported. + /// Gets a value indicating whether shuffle operations are supported. /// - public static bool SupportsRightShift + public static bool SupportsShuffleFloat + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Sse.IsSupported; + } + + /// + /// Gets a value indicating whether shuffle operations are supported. + /// + public static bool SupportsShuffleByte + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Ssse3.IsSupported || AdvSimd.Arm64.IsSupported; + } + + /// + /// Gets a value indicating whether right align operations are supported. + /// + public static bool SupportsRightAlign { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => Ssse3.IsSupported || AdvSimd.IsSupported; } + /// + /// Gets a value indicating whether right or left byte shift operations are supported. + /// + public static bool SupportsShiftByte + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Sse2.IsSupported || AdvSimd.IsSupported; + } + + /// + /// Creates a new vector by selecting values from an input vector using the control. + /// + /// The input vector from which values are selected. + /// The shuffle control byte. + /// The . + public static Vector128 Shuffle(Vector128 vector, [ConstantExpected] byte control) + { + if (Sse.IsSupported) + { + return Sse.Shuffle(vector, vector, control); + } + + ThrowUnreachableException(); + return default; + } + + /// + /// Creates a new vector by selecting values from an input vector using a set of indices. + /// + /// + /// The input vector from which values are selected. + /// + /// The per-element indices used to select a value from . + /// + /// + /// A new vector containing the values from selected by the given . + /// + public static Vector128 Shuffle(Vector128 vector, Vector128 indices) + { + if (Ssse3.IsSupported) + { + return Ssse3.Shuffle(vector, indices); + } + + if (AdvSimd.Arm64.IsSupported) + { + return AdvSimd.Arm64.VectorTableLookup(vector, indices); + } + + ThrowUnreachableException(); + return default; + } + /// /// Shifts a 128-bit value right by a specified number of bytes while shifting in zeros. /// @@ -80,7 +155,7 @@ internal static class Vector128Utilities /// The . public static Vector128 AlignRight(Vector128 left, Vector128 right, [ConstantExpected(Max = (byte)15)] byte mask) { - if (Sse3.IsSupported) + if (Ssse3.IsSupported) { return Ssse3.AlignRight(left, right, mask); } diff --git a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs new file mode 100644 index 0000000000..14fa24b315 --- /dev/null +++ b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs @@ -0,0 +1,78 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace SixLabors.ImageSharp.Common.Helpers; + +/// +/// Defines utility methods for that have either: +/// +/// Not yet been normalized in the runtime. +/// Produce codegen that is poorly optimized by the runtime. +/// +/// Should only be used if the intrinsics are available. +/// +internal static class Vector256Utilities +{ + /// + /// Gets a value indicating whether shuffle byte operations are supported. + /// + public static bool SupportsShuffleFloat + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Avx.IsSupported; + } + + /// + /// Gets a value indicating whether shuffle byte operations are supported. + /// + public static bool SupportsShuffleByte + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Avx2.IsSupported; + } + + /// + /// Creates a new vector by selecting values from an input vector using a set of indices. + /// + /// The input vector from which values are selected. + /// The shuffle control byte. + /// The . + public static Vector256 Shuffle(Vector256 vector, [ConstantExpected] byte control) + { + if (Avx.IsSupported) + { + return Avx.Shuffle(vector, vector, control); + } + + ThrowUnreachableException(); + return default; + } + + /// + /// Creates a new vector by selecting values from an input vector using a set of indices. + /// + /// The input vector from which values are selected. + /// + /// The per-element indices used to select a value from . + /// + /// The . + public static Vector256 Shuffle(Vector256 vector, Vector256 indices) + { + if (Avx2.IsSupported) + { + return Avx2.Shuffle(vector, indices); + } + + ThrowUnreachableException(); + return default; + } + + [DoesNotReturn] + private static void ThrowUnreachableException() => throw new UnreachableException(); +} diff --git a/src/ImageSharp/Common/Helpers/Vector512Utilities.cs b/src/ImageSharp/Common/Helpers/Vector512Utilities.cs new file mode 100644 index 0000000000..5488b40644 --- /dev/null +++ b/src/ImageSharp/Common/Helpers/Vector512Utilities.cs @@ -0,0 +1,80 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace SixLabors.ImageSharp.Common.Helpers; + +/// +/// Defines utility methods for that have either: +/// +/// Not yet been normalized in the runtime. +/// Produce codegen that is poorly optimized by the runtime. +/// +/// Should only be used if the intrinsics are available. +/// +internal static class Vector512Utilities +{ + /// + /// Gets a value indicating whether shuffle float operations are supported. + /// + public static bool SupportsShuffleFloat + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Avx512F.IsSupported; + } + + /// + /// Gets a value indicating whether shuffle byte operations are supported. + /// + public static bool SupportsShuffleByte + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Avx512BW.IsSupported; + } + + /// + /// Creates a new vector by selecting values from an input vector using the control. + /// + /// The input vector from which values are selected. + /// The shuffle control byte. + /// The . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Shuffle(Vector512 vector, [ConstantExpected] byte control) + { + if (Avx512F.IsSupported) + { + return Avx512F.Shuffle(vector, vector, control); + } + + ThrowUnreachableException(); + return default; + } + + /// + /// Creates a new vector by selecting values from an input vector using a set of indices. + /// + /// The input vector from which values are selected. + /// + /// The per-element indices used to select a value from . + /// + /// The . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Shuffle(Vector512 vector, Vector512 indices) + { + if (Avx512BW.IsSupported) + { + return Avx512BW.Shuffle(vector, indices); + } + + ThrowUnreachableException(); + return default; + } + + [DoesNotReturn] + private static void ThrowUnreachableException() => throw new UnreachableException(); +}