diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs b/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs
index c856267db..6b80845ed 100644
--- a/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs
+++ b/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs
@@ -1,36 +1,26 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
-// The JIT can detect and optimize rotation idioms ROTL (Rotate Left)
-// and ROTR (Rotate Right) emitting efficient CPU instructions:
-// https://github.com/dotnet/coreclr/pull/1830
+using System.Runtime.Intrinsics;
+
namespace SixLabors.ImageSharp;
///
-/// Defines the contract for methods that allow the shuffling of pixel components.
-/// Used for shuffling on platforms that do not support Hardware Intrinsics.
+/// Defines a stateless operation over packed pixel components.
///
internal interface IComponentShuffle
{
///
- /// Shuffles then slices 8-bit integers in
- /// using a byte control and store the results in .
- /// If successful, this method will reduce the length of length
- /// by the shuffle amount.
+ /// Reorders one packed pixel.
///
- /// The source span of bytes.
- /// The destination span of bytes.
- void ShuffleReduce(ref ReadOnlySpan source, ref Span destination);
+ /// The source components, with the first component in the least-significant byte.
+ /// The reordered packed components.
+ static abstract uint Invoke(uint source);
///
- /// Shuffle 8-bit integers in
- /// using the control and store the results in .
+ /// Reorders the packed pixels in a 128-bit vector.
///
- /// The source span of bytes.
- /// The destination span of bytes.
- ///
- /// Implementation can assume that source.Length is less or equal than destination.Length.
- /// Loops should iterate using source.Length.
- ///
- void Shuffle(ReadOnlySpan source, Span destination);
+ /// The source pixels.
+ /// The reordered pixels.
+ static abstract Vector128 Invoke(Vector128 source);
}
diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
index bbe14b099..d9d8e35fc 100644
--- a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
+++ b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
@@ -1,99 +1,82 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
-using System.Diagnostics.CodeAnalysis;
+using System.Buffers.Binary;
+using System.Numerics;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using static SixLabors.ImageSharp.SimdUtils;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
-///
+///
+/// Defines a stateless operation that reorders a three-component pixel after adding opaque alpha.
+///
internal interface IPad3Shuffle4 : IComponentShuffle
{
}
-internal readonly struct DefaultPad3Shuffle4([ConstantExpected] byte control) : IPad3Shuffle4
+///
+/// Preserves XYZ order and appends opaque W.
+///
+internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4
{
- public byte Control { get; } = control;
-
- [MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
-#pragma warning disable CA1857 // A constant is expected for the parameter
- => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, this.Control);
-#pragma warning restore CA1857 // A constant is expected for the parameter
-
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
- {
- ref byte sBase = ref MemoryMarshal.GetReference(source);
- ref byte dBase = ref MemoryMarshal.GetReference(destination);
-
- SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0);
+ public static uint Invoke(uint source) => source;
- for (nuint i = 0, j = 0; i < (uint)source.Length; i += 3, j += 4)
- {
- // Expanding 3-byte pixels to 4 bytes can overwrite the next source
- // triplet when spans overlap. Assemble the padded pixel first, then
- // shuffle from the staged uint.
- uint packed =
- Unsafe.Add(ref sBase, i + 0u) |
- ((uint)Unsafe.Add(ref sBase, i + 1u) << 8) |
- ((uint)Unsafe.Add(ref sBase, i + 2u) << 16) |
- 0xFF000000;
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source) => source;
+}
- ref byte pBase = ref Unsafe.As(ref packed);
+///
+/// Reorders padded XYZW components to WXYZ.
+///
+internal readonly struct WXYZPad3Shuffle4 : IPad3Shuffle4
+{
+ ///
+ [MethodImpl(InliningOptions.ShortMethod)]
+ public static uint Invoke(uint source) => BitOperations.RotateLeft(source, 8);
- Unsafe.Add(ref dBase, j + 0u) = Unsafe.Add(ref pBase, p0);
- Unsafe.Add(ref dBase, j + 1u) = Unsafe.Add(ref pBase, p1);
- Unsafe.Add(ref dBase, j + 2u) = Unsafe.Add(ref pBase, p2);
- Unsafe.Add(ref dBase, j + 3u) = Unsafe.Add(ref pBase, p3);
- }
- }
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14));
}
-internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4
+///
+/// Reorders padded XYZW components to WZYX.
+///
+internal readonly struct WZYXPad3Shuffle4 : IPad3Shuffle4
{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210);
+ public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source);
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
+}
+
+///
+/// Reorders padded XYZW components to ZYXW.
+///
+internal readonly struct ZYXWPad3Shuffle4 : IPad3Shuffle4
+{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ public static uint Invoke(uint source)
{
- ref byte sBase = ref MemoryMarshal.GetReference(source);
- ref byte dBase = ref MemoryMarshal.GetReference(destination);
-
- ref byte sEnd = ref Unsafe.Add(ref sBase, (uint)source.Length);
- ref byte sLoopEnd = ref Unsafe.Subtract(ref sEnd, 4);
-
- while (Unsafe.IsAddressLessThan(ref sBase, ref sLoopEnd))
- {
- // The fast scalar path reads one extra byte past the source triplet.
- // Keep that widened read in a local before writing the expanded pixel
- // so overlapping destinations cannot change what was read.
- uint packed = Unsafe.As(ref sBase) | 0xFF000000;
-
- Unsafe.As(ref dBase) = packed;
-
- sBase = ref Unsafe.Add(ref sBase, 3);
- dBase = ref Unsafe.Add(ref dBase, 4);
- }
-
- while (Unsafe.IsAddressLessThan(ref sBase, ref sEnd))
- {
- // The final triplet cannot use the widened read above, so assemble
- // the same padded uint byte-by-byte before the overlapping store.
- uint packed =
- Unsafe.Add(ref sBase, 0u) |
- ((uint)Unsafe.Add(ref sBase, 1u) << 8) |
- ((uint)Unsafe.Add(ref sBase, 2u) << 16) |
- 0xFF000000;
-
- Unsafe.As(ref dBase) = packed;
-
- sBase = ref Unsafe.Add(ref sBase, 3);
- dBase = ref Unsafe.Add(ref dBase, 4);
- }
+ // Preserve opaque W and Y while exchanging X and Z.
+ uint wy = source & 0xFF00FF00;
+ uint xz = source & 0x00FF00FF;
+ return wy | BitOperations.RotateLeft(xz, 16);
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
}
diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
index 3907df58c..eab5a41da 100644
--- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
+++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
@@ -1,51 +1,37 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
-using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using static SixLabors.ImageSharp.SimdUtils;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
-///
+///
+/// Identifies a stateless three-component shuffle operator.
+///
internal interface IShuffle3 : IComponentShuffle
{
}
-internal readonly struct DefaultShuffle3([ConstantExpected] byte control) : IShuffle3
+///
+/// Reorders XYZ components to ZYX.
+///
+internal readonly struct ZYXShuffle3 : IShuffle3
{
- public byte Control { get; } = control;
-
- [MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
-#pragma warning disable CA1857 // A constant is expected for the parameter
- => HwIntrinsics.Shuffle3Reduce(ref source, ref destination, this.Control);
-#pragma warning restore CA1857 // A constant is expected for the parameter
-
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ public static uint Invoke(uint source)
{
- ref byte sBase = ref MemoryMarshal.GetReference(source);
- ref byte dBase = ref MemoryMarshal.GetReference(destination);
-
- SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0);
-
- for (nuint i = 0; i < (uint)source.Length; i += 3)
- {
- // The scalar remainder can run in-place after the vector body. Load
- // the full 3-byte pixel into a register-sized value before stores so
- // channel swaps cannot corrupt later reads from the same pixel.
- uint packed =
- Unsafe.Add(ref sBase, i + 0u) |
- ((uint)Unsafe.Add(ref sBase, i + 1u) << 8) |
- ((uint)Unsafe.Add(ref sBase, i + 2u) << 16);
-
- ref byte pBase = ref Unsafe.As(ref packed);
-
- Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0);
- Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1);
- Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2);
- }
+ // Y is already centered; shift X and Z directly into each other's byte positions.
+ uint y = source & 0x0000FF00;
+ uint x = (source & 0x000000FF) << 16;
+ uint z = (source & 0x00FF0000) >> 16;
+ return x | y | z;
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
}
diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs
index 68f34efd7..56ea59df5 100644
--- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs
+++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs
@@ -2,183 +2,320 @@
// Licensed under the Six Labors Split License.
using System.Buffers.Binary;
-using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using static SixLabors.ImageSharp.SimdUtils;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
-///
+///
+/// Defines a stateless operation over one packed four-component pixel.
+///
internal interface IShuffle4 : IComponentShuffle
{
+ ///
+ /// Reorders the packed pixels in a 256-bit vector.
+ ///
+ /// The source pixels.
+ /// The reordered pixels.
+ static abstract Vector256 Invoke(Vector256 source);
+
+ ///
+ /// Reorders the packed pixels in a 512-bit vector.
+ ///
+ /// The source pixels.
+ /// The reordered pixels.
+ static abstract Vector512 Invoke(Vector512 source);
}
-internal readonly struct DefaultShuffle4([ConstantExpected] byte control) : IShuffle4
+///
+/// Reorders XYZW components to WXYZ.
+///
+internal readonly struct WXYZShuffle4 : IShuffle4
{
- public byte Control { get; } = control;
-
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
-#pragma warning disable CA1857 // A constant is expected for the parameter
- => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, this.Control);
-#pragma warning restore CA1857 // A constant is expected for the parameter
+ public static uint Invoke(uint source)
+ {
+ // source = [W Z Y X]
+ // ROTL(8, source) = [Z Y X W]
+ return BitOperations.RotateLeft(source, 8);
+ }
- [MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, CreateMask());
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 source)
{
- ref byte sBase = ref MemoryMarshal.GetReference(source);
- ref byte dBase = ref MemoryMarshal.GetReference(destination);
-
- SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0);
-
- for (nuint i = 0; i < (uint)source.Length; i += 4)
- {
- // The generic path may be used with source and destination pointing
- // at the same pixel. Load all channels first so subsequent stores
- // index only staged bytes, matching the specialized uint shuffles.
- uint packed = Unsafe.As(ref Unsafe.Add(ref sBase, i));
- ref byte pBase = ref Unsafe.As(ref packed);
-
- Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0);
- Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1);
- Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2);
- Unsafe.Add(ref dBase, i + 3u) = Unsafe.Add(ref pBase, p3);
- }
+ // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
+ Vector128 mask = CreateMask();
+ return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
-}
-internal readonly struct WXYZShuffle4 : IShuffle4
-{
- [MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle2103);
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 source)
+ => Vector512_.ShuffleNative(source, CreateMask512());
- [MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ ///
+ /// Creates the indices that rotate each XYZW pixel to WXYZ within one 128-bit lane.
+ ///
+ /// The pixel-local byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 CreateMask()
+ => Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14);
+
+ ///
+ /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
+ ///
+ /// The absolute byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 CreateMask512()
{
- ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source));
- ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
- uint n = (uint)source.Length / 4;
-
- for (nuint i = 0; i < n; i++)
- {
- uint packed = Unsafe.Add(ref sBase, i);
-
- // packed = [W Z Y X]
- // ROTL(8, packed) = [Z Y X W]
- Unsafe.Add(ref dBase, i) = (packed << 8) | (packed >> 24);
- }
+ // Native vpshufb ignores the lane offsets, while Vector512.Shuffle treats the indices as
+ // absolute. Encoding both meanings keeps the AVX-512 and managed fallback results identical.
+ return Vector512.Create(
+ 0x0605040702010003UL,
+ 0x0E0D0C0F0A09080BUL,
+ 0x1615141712111013UL,
+ 0x1E1D1C1F1A19181BUL,
+ 0x2625242722212023UL,
+ 0x2E2D2C2F2A29282BUL,
+ 0x3635343732313033UL,
+ 0x3E3D3C3F3A39383BUL).AsByte();
}
}
+///
+/// Reorders XYZW components to WZYX.
+///
internal readonly struct WZYXShuffle4 : IShuffle4
{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0123);
+ public static uint Invoke(uint source)
+ {
+ // Reversing the integer's endianness also reverses the four byte components.
+ return BinaryPrimitives.ReverseEndianness(source);
+ }
- [MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, CreateMask());
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 source)
{
- ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source));
- ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
- uint n = (uint)source.Length / 4;
-
- for (nuint i = 0; i < n; i++)
- {
- uint packed = Unsafe.Add(ref sBase, i);
-
- // packed = [W Z Y X]
- // REVERSE(packedArgb) = [X Y Z W]
- Unsafe.Add(ref dBase, i) = BinaryPrimitives.ReverseEndianness(packed);
- }
+ Vector128 mask = CreateMask();
+ return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 source)
+ => Vector512_.ShuffleNative(source, CreateMask512());
+
+ ///
+ /// Creates the indices that reverse each XYZW pixel to WZYX within one 128-bit lane.
+ ///
+ /// The pixel-local byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 CreateMask()
+ => Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12);
+
+ ///
+ /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
+ ///
+ /// The absolute byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 CreateMask512()
+ => Vector512.Create(
+ 0x0405060700010203UL,
+ 0x0C0D0E0F08090A0BUL,
+ 0x1415161710111213UL,
+ 0x1C1D1E1F18191A1BUL,
+ 0x2425262720212223UL,
+ 0x2C2D2E2F28292A2BUL,
+ 0x3435363730313233UL,
+ 0x3C3D3E3F38393A3BUL).AsByte();
}
+///
+/// Reorders XYZW components to YZWX.
+///
internal readonly struct YZWXShuffle4 : IShuffle4
{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0321);
+ public static uint Invoke(uint source)
+ {
+ // source = [W Z Y X]
+ // ROTR(8, source) = [X W Z Y]
+ return BitOperations.RotateRight(source, 8);
+ }
- [MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, CreateMask());
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 source)
{
- ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source));
- ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
- uint n = (uint)source.Length / 4;
-
- for (nuint i = 0; i < n; i++)
- {
- uint packed = Unsafe.Add(ref sBase, i);
-
- // packed = [W Z Y X]
- // ROTR(8, packedArgb) = [Y Z W X]
- Unsafe.Add(ref dBase, i) = BitOperations.RotateRight(packed, 8);
- }
+ Vector128 mask = CreateMask();
+ return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 source)
+ => Vector512_.ShuffleNative(source, CreateMask512());
+
+ ///
+ /// Creates the indices that rotate each XYZW pixel to YZWX within one 128-bit lane.
+ ///
+ /// The pixel-local byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 CreateMask()
+ => Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12);
+
+ ///
+ /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
+ ///
+ /// The absolute byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 CreateMask512()
+ => Vector512.Create(
+ 0x0407060500030201UL,
+ 0x0C0F0E0D080B0A09UL,
+ 0x1417161510131211UL,
+ 0x1C1F1E1D181B1A19UL,
+ 0x2427262520232221UL,
+ 0x2C2F2E2D282B2A29UL,
+ 0x3437363530333231UL,
+ 0x3C3F3E3D383B3A39UL).AsByte();
}
+///
+/// Reorders XYZW components to ZYXW.
+///
internal readonly struct ZYXWShuffle4 : IShuffle4
{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3012);
+ public static uint Invoke(uint source)
+ {
+ // Preserve W and Y while rotating the masked X/Z bytes into each other's positions.
+ uint wy = source & 0xFF00FF00;
+ uint xz = source & 0x00FF00FF;
+ return wy | BitOperations.RotateLeft(xz, 16);
+ }
- [MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, CreateMask());
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 source)
{
- ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source));
- ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
- uint n = (uint)source.Length / 4;
-
- for (nuint i = 0; i < n; i++)
- {
- uint packed = Unsafe.Add(ref sBase, i);
-
- // packed = [W Z Y X]
- // tmp1 = [W 0 Y 0]
- // tmp2 = [0 Z 0 X]
- // tmp3=ROTL(16, tmp2) = [0 X 0 Z]
- // tmp1 + tmp3 = [W X Y Z]
- uint tmp1 = packed & 0xFF00FF00;
- uint tmp2 = packed & 0x00FF00FF;
- uint tmp3 = BitOperations.RotateLeft(tmp2, 16);
-
- Unsafe.Add(ref dBase, i) = tmp1 + tmp3;
- }
+ Vector128 mask = CreateMask();
+ return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 source)
+ => Vector512_.ShuffleNative(source, CreateMask512());
+
+ ///
+ /// Creates the indices that exchange X and Z in each XYZW pixel within one 128-bit lane.
+ ///
+ /// The pixel-local byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 CreateMask()
+ => Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15);
+
+ ///
+ /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
+ ///
+ /// The absolute byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 CreateMask512()
+ => Vector512.Create(
+ 0x0704050603000102UL,
+ 0x0F0C0D0E0B08090AUL,
+ 0x1714151613101112UL,
+ 0x1F1C1D1E1B18191AUL,
+ 0x2724252623202122UL,
+ 0x2F2C2D2E2B28292AUL,
+ 0x3734353633303132UL,
+ 0x3F3C3D3E3B38393AUL).AsByte();
}
+///
+/// Reorders XYZW components to XWZY.
+///
internal readonly struct XWZYShuffle4 : IShuffle4
{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle1230);
+ public static uint Invoke(uint source)
+ {
+ // Preserve X and Z while rotating the masked Y/W bytes into each other's positions.
+ uint xz = source & 0x00FF00FF;
+ uint yw = source & 0xFF00FF00;
+ return xz | BitOperations.RotateLeft(yw, 16);
+ }
- [MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, CreateMask());
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 Invoke(Vector256 source)
{
- ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source));
- ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
- uint n = (uint)source.Length / 4;
-
- for (nuint i = 0; i < n; i++)
- {
- uint packed = Unsafe.Add(ref sBase, i);
-
- // packed = [W Z Y X]
- // tmp1 = [0 Z 0 X]
- // tmp2 = [W 0 Y 0]
- // tmp3=ROTL(16, tmp2) = [Y 0 W 0]
- // tmp1 + tmp3 = [Y Z W X]
- uint tmp1 = packed & 0x00FF00FF;
- uint tmp2 = packed & 0xFF00FF00;
- uint tmp3 = BitOperations.RotateLeft(tmp2, 16);
-
- Unsafe.Add(ref dBase, i) = tmp1 + tmp3;
- }
+ Vector128 mask = CreateMask();
+ return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Invoke(Vector512 source)
+ => Vector512_.ShuffleNative(source, CreateMask512());
+
+ ///
+ /// Creates the indices that exchange Y and W in each XYZW pixel within one 128-bit lane.
+ ///
+ /// The pixel-local byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector128 CreateMask()
+ => Vector128.Create((byte)0, 3, 2, 1, 4, 7, 6, 5, 8, 11, 10, 9, 12, 15, 14, 13);
+
+ ///
+ /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
+ ///
+ /// The absolute byte shuffle indices.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector512 CreateMask512()
+ => Vector512.Create(
+ 0x0506070401020300UL,
+ 0x0D0E0F0C090A0B08UL,
+ 0x1516171411121310UL,
+ 0x1D1E1F1C191A1B18UL,
+ 0x2526272421222320UL,
+ 0x2D2E2F2C292A2B28UL,
+ 0x3536373431323330UL,
+ 0x3D3E3F3C393A3B38UL).AsByte();
}
diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
index 613406167..17ec503e5 100644
--- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
+++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
@@ -1,101 +1,85 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
-using System.Diagnostics.CodeAnalysis;
+using System.Buffers.Binary;
+using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
-using static SixLabors.ImageSharp.SimdUtils;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
-///
+///
+/// Defines a stateless operation that reorders four packed components before retaining three.
+///
internal interface IShuffle4Slice3 : IComponentShuffle
{
}
-internal readonly struct DefaultShuffle4Slice3([ConstantExpected] byte control) : IShuffle4Slice3
+///
+/// Preserves XYZ order and discards W.
+///
+internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3
{
- public byte Control { get; } = control;
-
- [MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
-#pragma warning disable CA1857 // A constant is expected for the parameter
- => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, this.Control);
-#pragma warning restore CA1857 // A constant is expected for the parameter
-
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
- {
- ref byte sBase = ref MemoryMarshal.GetReference(source);
- ref byte dBase = ref MemoryMarshal.GetReference(destination);
+ public static uint Invoke(uint source) => source;
- SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0);
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source) => source;
+}
- for (nuint i = 0, j = 0; i < (uint)destination.Length; i += 3, j += 4)
- {
- // Shrinking 4-byte pixels to 3 bytes can still be called in-place by
- // tail code. Read the complete source pixel first, then write only
- // the requested channels into the destination triplet.
- uint packed = Unsafe.As(ref Unsafe.Add(ref sBase, j));
- ref byte pBase = ref Unsafe.As(ref packed);
+///
+/// Reorders XYZW components to YZW before discarding X.
+///
+internal readonly struct YZWXShuffle4Slice3 : IShuffle4Slice3
+{
+ ///
+ [MethodImpl(InliningOptions.ShortMethod)]
+ public static uint Invoke(uint source) => BitOperations.RotateRight(source, 8);
- Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0);
- Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1);
- Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2);
- }
- }
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12));
}
-internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3
+///
+/// Reorders XYZW components to WZY before discarding X.
+///
+internal readonly struct WZYXShuffle4Slice3 : IShuffle4Slice3
{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination)
- => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210);
+ public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source);
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
+}
+///
+/// Reorders XYZW components to ZYX before discarding W.
+///
+internal readonly struct ZYXWShuffle4Slice3 : IShuffle4Slice3
+{
+ ///
[MethodImpl(InliningOptions.ShortMethod)]
- public void Shuffle(ReadOnlySpan source, Span destination)
+ public static uint Invoke(uint source)
{
- ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source));
- ref Byte3 dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
-
- nint n = (nint)(uint)source.Length / 4;
- nint m = Numerics.Modulo4(n);
- nint u = n - m;
-
- ref uint sLoopEnd = ref Unsafe.Add(ref sBase, u);
- ref uint sEnd = ref Unsafe.Add(ref sBase, n);
-
- while (Unsafe.IsAddressLessThan(ref sBase, ref sLoopEnd))
- {
- // Stage the four source pixels before the 3-byte stores. Even
- // though this path preserves XYZ order, the packed loads must happen
- // before destination writes when the spans overlap.
- uint packed0 = Unsafe.Add(ref sBase, 0u);
- uint packed1 = Unsafe.Add(ref sBase, 1u);
- uint packed2 = Unsafe.Add(ref sBase, 2u);
- uint packed3 = Unsafe.Add(ref sBase, 3u);
-
- Unsafe.Add(ref dBase, 0u) = Unsafe.As(ref packed0);
- Unsafe.Add(ref dBase, 1u) = Unsafe.As(ref packed1);
- Unsafe.Add(ref dBase, 2u) = Unsafe.As(ref packed2);
- Unsafe.Add(ref dBase, 3u) = Unsafe.As(ref packed3);
-
- sBase = ref Unsafe.Add(ref sBase, 4);
- dBase = ref Unsafe.Add(ref dBase, 4);
- }
-
- while (Unsafe.IsAddressLessThan(ref sBase, ref sEnd))
- {
- // Same overlap rule as the unrolled loop: take the 4-byte source
- // pixel before storing the 3-byte destination value.
- uint packed = Unsafe.Add(ref sBase, 0u);
-
- Unsafe.Add(ref dBase, 0u) = Unsafe.As(ref packed);
-
- sBase = ref Unsafe.Add(ref sBase, 1);
- dBase = ref Unsafe.Add(ref dBase, 1);
- }
+ // Preserve W and Y while exchanging X and Z; W is subsequently discarded.
+ uint wy = source & 0xFF00FF00;
+ uint xz = source & 0x00FF00FF;
+ return wy | BitOperations.RotateLeft(xz, 16);
}
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector128 Invoke(Vector128 source)
+ => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
}
[StructLayout(LayoutKind.Explicit, Size = 3)]
diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs b/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs
index 8b2baec21..e709cc8d4 100644
--- a/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs
+++ b/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs
@@ -6,6 +6,8 @@ using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
@@ -42,22 +44,104 @@ internal static partial class SimdUtils
/// The type of shuffle struct.
/// The source span of bytes.
/// The destination span of bytes.
- /// The type of shuffle to perform.
[MethodImpl(InliningOptions.ShortMethod)]
public static void Shuffle4(
ReadOnlySpan source,
- Span destination,
- TShuffle shuffle)
+ Span destination)
where TShuffle : struct, IShuffle4
{
VerifyShuffle4SpanInput(source, destination);
- shuffle.ShuffleReduce(ref source, ref destination);
+ ref byte sourceBase = ref MemoryMarshal.GetReference(source);
+ ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
+ int length = source.Length;
+ int i = 0;
- // Deal with the remainder:
- if (source.Length > 0)
+ // The same offset flows through descending widths. This keeps a single traversal while
+ // allowing a row that is not a multiple of the widest register to retain a vectorized tail.
+ if (Vector512.IsHardwareAccelerated)
+ {
+ int fourVectorsFromEnd = length - (Vector512.Count * 4);
+
+ for (; i <= fourVectorsFromEnd; i += Vector512.Count * 4)
+ {
+ // Four independent vectors amortize loop control and expose enough work for the CPU
+ // to overlap loads, byte shuffles, and stores without changing pixel ordering.
+ TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i))
+ .StoreUnsafe(ref destinationBase, (nuint)i);
+ TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + Vector512.Count)))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + Vector512.Count));
+ TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512.Count * 2))))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512.Count * 2)));
+ TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512.Count * 3))))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512.Count * 3)));
+ }
+
+ int oneVectorFromEnd = length - Vector512.Count;
+
+ for (; i <= oneVectorFromEnd; i += Vector512.Count)
+ {
+ TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i))
+ .StoreUnsafe(ref destinationBase, (nuint)i);
+ }
+ }
+
+ if (Vector256.IsHardwareAccelerated)
+ {
+ int fourVectorsFromEnd = length - (Vector256.Count * 4);
+
+ for (; i <= fourVectorsFromEnd; i += Vector256.Count * 4)
+ {
+ TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i))
+ .StoreUnsafe(ref destinationBase, (nuint)i);
+ TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + Vector256.Count)))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + Vector256.Count));
+ TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256.Count * 2))))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256.Count * 2)));
+ TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256.Count * 3))))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256.Count * 3)));
+ }
+
+ int oneVectorFromEnd = length - Vector256.Count;
+
+ for (; i <= oneVectorFromEnd; i += Vector256.Count)
+ {
+ TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i))
+ .StoreUnsafe(ref destinationBase, (nuint)i);
+ }
+ }
+
+ if (Vector128.IsHardwareAccelerated)
{
- shuffle.Shuffle(source, destination);
+ int fourVectorsFromEnd = length - (Vector128.Count * 4);
+
+ for (; i <= fourVectorsFromEnd; i += Vector128.Count * 4)
+ {
+ TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i))
+ .StoreUnsafe(ref destinationBase, (nuint)i);
+ TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + Vector128.Count)))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + Vector128.Count));
+ TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128.Count * 2))))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128.Count * 2)));
+ TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128.Count * 3))))
+ .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128.Count * 3)));
+ }
+
+ int oneVectorFromEnd = length - Vector128.Count;
+
+ for (; i <= oneVectorFromEnd; i += Vector128.Count)
+ {
+ TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i))
+ .StoreUnsafe(ref destinationBase, (nuint)i);
+ }
+ }
+
+ // The vector cascade leaves fewer than four pixels. A full uint load keeps each pixel
+ // in a register while the closed operator resolves to its rotate, reverse, or mask sequence.
+ for (; i < length; i += 4)
+ {
+ uint packed = Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)i));
+ Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)i)) = TShuffle.Invoke(packed);
}
}
@@ -68,23 +152,112 @@ internal static partial class SimdUtils
/// The type of shuffle struct.
/// The source span of bytes.
/// The destination span of bytes.
- /// The type of shuffle to perform.
[MethodImpl(InliningOptions.ShortMethod)]
public static void Shuffle3(
ReadOnlySpan source,
- Span destination,
- TShuffle shuffle)
+ Span destination)
where TShuffle : struct, IShuffle3
{
- // Source length should be smaller than destination length, and divisible by 3.
VerifyShuffle3SpanInput(source, destination);
- shuffle.ShuffleReduce(ref source, ref destination);
+ ref byte sourceBase = ref MemoryMarshal.GetReference(source);
+ ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
+ int length = source.Length;
+ int i = 0;
- // Deal with the remainder:
- if (source.Length > 0)
+ if (Vector128.IsHardwareAccelerated)
+ {
+ // Each group contains sixteen XYZ pixels in three registers. The pad mask expands
+ // four triplets per register to XYZW, with 0x80 selecting zero for the temporary W lane.
+ Vector128 padMask = Vector128.Create(
+ (byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
+
+ // After the operator has reordered padded pixels, these masks remove every temporary
+ // W lane and repack the four registers into three contiguous XYZ destination registers.
+ Vector128 sliceMask = Vector128.Create(
+ (byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
+ Vector128 sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12);
+ ref Vector128 sourceVectors = ref Unsafe.As>(ref sourceBase);
+ ref Vector128 destinationVectors = ref Unsafe.As>(ref destinationBase);
+ nuint sourceVectorCount = (uint)length / (uint)Vector128.Count;
+ nuint vectorIndex = 0;
+
+ for (; vectorIndex + 2 < sourceVectorCount; vectorIndex += 3)
+ {
+ // Realign the three source registers into four registers holding four complete
+ // triplets apiece. All source registers are captured before any destination store.
+ ref Vector128 source0 = ref Unsafe.Add(ref sourceVectors, vectorIndex);
+ Vector128 v0 = source0;
+ Vector128 v1 = Unsafe.Add(ref source0, 1);
+ Vector128 v2 = Unsafe.Add(ref source0, 2);
+ Vector128 v3 = Vector128_.ShiftRightBytesInVector(v2, 4);
+
+ v2 = Vector128_.AlignRight(v2, v1, 8);
+ v1 = Vector128_.AlignRight(v1, v0, 12);
+
+ v0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask));
+ v1 = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask));
+ v2 = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask));
+ v3 = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask));
+
+ v0 = Vector128_.ShuffleNative(v0, sliceEndMask);
+ v1 = Vector128_.ShuffleNative(v1, sliceMask);
+ v2 = Vector128_.ShuffleNative(v2, sliceEndMask);
+ v3 = Vector128_.ShuffleNative(v3, sliceMask);
+
+ Vector128 destination0 = Vector128_.AlignRight(v1, v0, 4);
+ Vector128 destination2 = Vector128_.AlignRight(v3, v2, 12);
+ v1 = Vector128_.ShiftLeftBytesInVector(v1, 4);
+ v2 = Vector128_.ShiftRightBytesInVector(v2, 4);
+ Vector128 destination1 = Vector128_.AlignRight(v2, v1, 8);
+
+ ref Vector128 destination0Ref = ref Unsafe.Add(ref destinationVectors, vectorIndex);
+ destination0Ref = destination0;
+ Unsafe.Add(ref destination0Ref, 1) = destination1;
+ Unsafe.Add(ref destination0Ref, 2) = destination2;
+ }
+
+ i = (int)(vectorIndex * (uint)Vector128.Count);
+ int oneTailVectorFromEnd = length - Vector128.Count;
+
+ for (; i <= oneTailVectorFromEnd; i += 12)
+ {
+ // A single readable register contains four complete triplets plus four bytes from
+ // the following pixels. The pad mask ignores those extra bytes before the operator
+ // runs, and the slice mask packs the four results into the low twelve bytes.
+ Vector128 result = Vector128.LoadUnsafe(ref sourceBase, (nuint)i);
+ result = Vector128_.ShuffleNative(result, padMask);
+ result = TShuffle.Invoke(result);
+ result = Vector128_.ShuffleNative(result, sliceMask);
+
+ // Store exactly twelve bytes so an in-place shuffle does not overwrite the next
+ // source triplet captured by the following iteration.
+ Unsafe.As>(ref Unsafe.Add(ref destinationBase, (nuint)i)) = result.GetLower();
+ Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)(i + 8))) = result.AsUInt32().GetElement(2);
+ }
+ }
+
+ int widenedReadEnd = length - 3;
+
+ for (; i < widenedReadEnd; i += 3)
{
- shuffle.Shuffle(source, destination);
+ // The fourth byte belongs to the following pixel, but the operator only contributes the
+ // low three result bytes. This unaligned read replaces three dependent byte loads safely.
+ uint packed = Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)i));
+ uint shuffled = TShuffle.Invoke(packed);
+ Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)i)) = Unsafe.As(ref shuffled);
+ }
+
+ if (i < length)
+ {
+ // The final triplet has no fourth readable byte, so construct only this terminal pixel.
+ uint packed =
+ Unsafe.Add(ref sourceBase, (nuint)i) |
+ ((uint)Unsafe.Add(ref sourceBase, (nuint)(i + 1)) << 8) |
+ ((uint)Unsafe.Add(ref sourceBase, (nuint)(i + 2)) << 16);
+
+ uint shuffled = TShuffle.Invoke(packed);
+ Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)i)) = Unsafe.As(ref shuffled);
}
}
@@ -95,22 +268,78 @@ internal static partial class SimdUtils
/// The type of shuffle struct.
/// The source span of bytes.
/// The destination span of bytes.
- /// The type of shuffle to perform.
[MethodImpl(InliningOptions.ShortMethod)]
public static void Pad3Shuffle4(
ReadOnlySpan source,
- Span destination,
- TShuffle shuffle)
+ Span destination)
where TShuffle : struct, IPad3Shuffle4
{
VerifyPad3Shuffle4SpanInput(source, destination);
- shuffle.ShuffleReduce(ref source, ref destination);
+ ref byte sourceBase = ref MemoryMarshal.GetReference(source);
+ ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
+ int sourceLength = source.Length;
+ int sourceOffset = 0;
+ int destinationOffset = 0;
- // Deal with the remainder:
- if (source.Length > 0)
+ if (Vector128.IsHardwareAccelerated)
+ {
+ // The fixed mask expands four XYZ triplets to four XYZW pixels. The zeroed W bytes
+ // are then filled with opaque alpha before the selected operator reorders each pixel.
+ Vector128 padMask = Vector128.Create(
+ (byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
+ Vector128 opaqueAlpha = Vector128.Create(0xFF000000FF000000UL).AsByte();
+ ref Vector128 sourceVectors = ref Unsafe.As>(ref sourceBase);
+ ref Vector128 destinationVectors = ref Unsafe.As>(ref destinationBase);
+ nuint sourceVectorCount = (uint)sourceLength / (uint)Vector128.Count;
+ nuint sourceVectorIndex = 0;
+ nuint destinationVectorIndex = 0;
+
+ for (; sourceVectorIndex + 2 < sourceVectorCount;
+ sourceVectorIndex += 3, destinationVectorIndex += 4)
+ {
+ // Three source registers contain sixteen packed triplets. Aligning at 12, 8, and
+ // 4-byte boundaries produces four registers whose low twelve bytes each hold four pixels.
+ ref Vector128 source0 = ref Unsafe.Add(ref sourceVectors, sourceVectorIndex);
+ Vector128 v0 = source0;
+ Vector128 v1 = Unsafe.Add(ref source0, 1);
+ Vector128 v2 = Unsafe.Add(ref source0, 2);
+ Vector128 v3 = Vector128_.ShiftRightBytesInVector(v2, 4);
+
+ v2 = Vector128_.AlignRight(v2, v1, 8);
+ v1 = Vector128_.AlignRight(v1, v0, 12);
+
+ ref Vector128 destination0 = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex);
+ destination0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask) | opaqueAlpha);
+ Unsafe.Add(ref destination0, 1) = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask) | opaqueAlpha);
+ Unsafe.Add(ref destination0, 2) = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask) | opaqueAlpha);
+ Unsafe.Add(ref destination0, 3) = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask) | opaqueAlpha);
+ }
+
+ sourceOffset = (int)(sourceVectorIndex * (uint)Vector128.Count);
+ destinationOffset = (int)(destinationVectorIndex * (uint)Vector128.Count);
+ }
+
+ int widenedReadEnd = sourceLength - 3;
+
+ for (; sourceOffset < widenedReadEnd; sourceOffset += 3, destinationOffset += 4)
+ {
+ // The widened load intentionally includes the next pixel's first byte. Replacing that
+ // high byte with opaque alpha yields the complete XYZW value with one unaligned read.
+ uint packed = Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset)) | 0xFF000000;
+ Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = TShuffle.Invoke(packed);
+ }
+
+ if (sourceOffset < sourceLength)
{
- shuffle.Shuffle(source, destination);
+ // The final triplet cannot use the widened load because no following byte is in range.
+ uint packed =
+ Unsafe.Add(ref sourceBase, (nuint)sourceOffset) |
+ ((uint)Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 1)) << 8) |
+ ((uint)Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 2)) << 16) |
+ 0xFF000000;
+
+ Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = TShuffle.Invoke(packed);
}
}
@@ -121,22 +350,101 @@ internal static partial class SimdUtils
/// The type of shuffle struct.
/// The source span of bytes.
/// The destination span of bytes.
- /// The type of shuffle to perform.
[MethodImpl(InliningOptions.ShortMethod)]
public static void Shuffle4Slice3(
ReadOnlySpan source,
- Span destination,
- TShuffle shuffle)
+ Span destination)
where TShuffle : struct, IShuffle4Slice3
{
VerifyShuffle4Slice3SpanInput(source, destination);
- shuffle.ShuffleReduce(ref source, ref destination);
+ ref byte sourceBase = ref MemoryMarshal.GetReference(source);
+ ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
+ int sourceLength = source.Length;
+ int sourceOffset = 0;
+ int destinationOffset = 0;
- // Deal with the remainder:
- if (source.Length > 0)
+ if (Vector128.IsHardwareAccelerated)
+ {
+ // Each operator first places the three retained components in the low bytes of every
+ // four-byte pixel. These masks then delete the fourth byte and compact sixteen pixels.
+ Vector128 sliceMask = Vector128.Create(
+ (byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
+ Vector128 sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12);
+ ref Vector128 sourceVectors = ref Unsafe.As>(ref sourceBase);
+ ref Vector128 destinationVectors = ref Unsafe.As>(ref destinationBase);
+ nuint sourceVectorCount = (uint)sourceLength / (uint)Vector128.Count;
+ nuint sourceVectorIndex = 0;
+ nuint destinationVectorIndex = 0;
+
+ for (; sourceVectorIndex + 3 < sourceVectorCount;
+ sourceVectorIndex += 4, destinationVectorIndex += 3)
+ {
+ // Load and transform all sixteen source pixels before writing the shorter output group.
+ // This preserves forward progress when source and destination begin at the same address.
+ ref Vector128 source0 = ref Unsafe.Add(ref sourceVectors, sourceVectorIndex);
+ Vector128 v0 = TShuffle.Invoke(source0);
+ Vector128 v1 = TShuffle.Invoke(Unsafe.Add(ref source0, 1));
+ Vector128 v2 = TShuffle.Invoke(Unsafe.Add(ref source0, 2));
+ Vector128 v3 = TShuffle.Invoke(Unsafe.Add(ref source0, 3));
+
+ v0 = Vector128_.ShuffleNative(v0, sliceEndMask);
+ v1 = Vector128_.ShuffleNative(v1, sliceMask);
+ v2 = Vector128_.ShuffleNative(v2, sliceEndMask);
+ v3 = Vector128_.ShuffleNative(v3, sliceMask);
+
+ Vector128 destination0 = Vector128_.AlignRight(v1, v0, 4);
+ Vector128 destination2 = Vector128_.AlignRight(v3, v2, 12);
+ v1 = Vector128_.ShiftLeftBytesInVector(v1, 4);
+ v2 = Vector128_.ShiftRightBytesInVector(v2, 4);
+ Vector128 destination1 = Vector128_.AlignRight(v2, v1, 8);
+
+ ref Vector128 destination0Ref = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex);
+ destination0Ref = destination0;
+ Unsafe.Add(ref destination0Ref, 1) = destination1;
+ Unsafe.Add(ref destination0Ref, 2) = destination2;
+ }
+
+ sourceOffset = (int)(sourceVectorIndex * (uint)Vector128.Count);
+ destinationOffset = (int)(destinationVectorIndex * (uint)Vector128