From 5a2fbbe795074969b3710dcad6462f6dacfe3651 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sat, 25 Jul 2026 07:38:24 +1000 Subject: [PATCH] Normalize packed pixel conversion shuffles --- .../Helpers/Shuffle/IComponentShuffle.cs | 32 +- .../Common/Helpers/Shuffle/IPad3Shuffle4.cs | 131 +++--- .../Common/Helpers/Shuffle/IShuffle3.cs | 56 +-- .../Common/Helpers/Shuffle/IShuffle4.cs | 395 ++++++++++++------ .../Common/Helpers/Shuffle/IShuffle4Slice3.cs | 130 +++--- .../Common/Helpers/SimdUtils.Shuffle.cs | 366 ++++++++++++++-- .../PixelFormats/Utils/PixelConverter.cs | 60 +-- .../Bulk/Pad3Shuffle4Channel.cs | 5 +- .../Bulk/Shuffle3Channel.cs | 3 +- .../Bulk/Shuffle4Slice3Channel.cs | 5 +- .../Bulk/ShuffleByte4Channel.cs | 2 +- .../PixelConversion/PackedPixelConversion.cs | 222 ++++++++++ .../PackedPixelConversionAssembly.cs | 128 ++++++ .../Common/SimdUtilsTests.Shuffle.cs | 100 +---- 14 files changed, 1157 insertions(+), 478 deletions(-) create mode 100644 tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversion.cs create mode 100644 tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs b/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs index c856267db..6b80845ed 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs @@ -1,36 +1,26 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -// The JIT can detect and optimize rotation idioms ROTL (Rotate Left) -// and ROTR (Rotate Right) emitting efficient CPU instructions: -// https://github.com/dotnet/coreclr/pull/1830 +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp; /// -/// Defines the contract for methods that allow the shuffling of pixel components. -/// Used for shuffling on platforms that do not support Hardware Intrinsics. +/// Defines a stateless operation over packed pixel components. /// internal interface IComponentShuffle { /// - /// Shuffles then slices 8-bit integers in - /// using a byte control and store the results in . - /// If successful, this method will reduce the length of length - /// by the shuffle amount. + /// Reorders one packed pixel. /// - /// The source span of bytes. - /// The destination span of bytes. - void ShuffleReduce(ref ReadOnlySpan source, ref Span destination); + /// The source components, with the first component in the least-significant byte. + /// The reordered packed components. + static abstract uint Invoke(uint source); /// - /// Shuffle 8-bit integers in - /// using the control and store the results in . + /// Reorders the packed pixels in a 128-bit vector. /// - /// The source span of bytes. - /// The destination span of bytes. - /// - /// Implementation can assume that source.Length is less or equal than destination.Length. - /// Loops should iterate using source.Length. - /// - void Shuffle(ReadOnlySpan source, Span destination); + /// The source pixels. + /// The reordered pixels. + static abstract Vector128 Invoke(Vector128 source); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs index bbe14b099..d9d8e35fc 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs @@ -1,99 +1,82 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Diagnostics.CodeAnalysis; +using System.Buffers.Binary; +using System.Numerics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using static SixLabors.ImageSharp.SimdUtils; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp; -/// +/// +/// Defines a stateless operation that reorders a three-component pixel after adding opaque alpha. +/// internal interface IPad3Shuffle4 : IComponentShuffle { } -internal readonly struct DefaultPad3Shuffle4([ConstantExpected] byte control) : IPad3Shuffle4 +/// +/// Preserves XYZ order and appends opaque W. +/// +internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4 { - public byte Control { get; } = control; - - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) -#pragma warning disable CA1857 // A constant is expected for the parameter - => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, this.Control); -#pragma warning restore CA1857 // A constant is expected for the parameter - + /// [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(destination); - - SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0); + public static uint Invoke(uint source) => source; - for (nuint i = 0, j = 0; i < (uint)source.Length; i += 3, j += 4) - { - // Expanding 3-byte pixels to 4 bytes can overwrite the next source - // triplet when spans overlap. Assemble the padded pixel first, then - // shuffle from the staged uint. - uint packed = - Unsafe.Add(ref sBase, i + 0u) | - ((uint)Unsafe.Add(ref sBase, i + 1u) << 8) | - ((uint)Unsafe.Add(ref sBase, i + 2u) << 16) | - 0xFF000000; + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) => source; +} - ref byte pBase = ref Unsafe.As(ref packed); +/// +/// Reorders padded XYZW components to WXYZ. +/// +internal readonly struct WXYZPad3Shuffle4 : IPad3Shuffle4 +{ + /// + [MethodImpl(InliningOptions.ShortMethod)] + public static uint Invoke(uint source) => BitOperations.RotateLeft(source, 8); - Unsafe.Add(ref dBase, j + 0u) = Unsafe.Add(ref pBase, p0); - Unsafe.Add(ref dBase, j + 1u) = Unsafe.Add(ref pBase, p1); - Unsafe.Add(ref dBase, j + 2u) = Unsafe.Add(ref pBase, p2); - Unsafe.Add(ref dBase, j + 3u) = Unsafe.Add(ref pBase, p3); - } - } + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14)); } -internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4 +/// +/// Reorders padded XYZW components to WZYX. +/// +internal readonly struct WZYXPad3Shuffle4 : IPad3Shuffle4 { + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210); + public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); +} + +/// +/// Reorders padded XYZW components to ZYXW. +/// +internal readonly struct ZYXWPad3Shuffle4 : IPad3Shuffle4 +{ + /// [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + public static uint Invoke(uint source) { - ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(destination); - - ref byte sEnd = ref Unsafe.Add(ref sBase, (uint)source.Length); - ref byte sLoopEnd = ref Unsafe.Subtract(ref sEnd, 4); - - while (Unsafe.IsAddressLessThan(ref sBase, ref sLoopEnd)) - { - // The fast scalar path reads one extra byte past the source triplet. - // Keep that widened read in a local before writing the expanded pixel - // so overlapping destinations cannot change what was read. - uint packed = Unsafe.As(ref sBase) | 0xFF000000; - - Unsafe.As(ref dBase) = packed; - - sBase = ref Unsafe.Add(ref sBase, 3); - dBase = ref Unsafe.Add(ref dBase, 4); - } - - while (Unsafe.IsAddressLessThan(ref sBase, ref sEnd)) - { - // The final triplet cannot use the widened read above, so assemble - // the same padded uint byte-by-byte before the overlapping store. - uint packed = - Unsafe.Add(ref sBase, 0u) | - ((uint)Unsafe.Add(ref sBase, 1u) << 8) | - ((uint)Unsafe.Add(ref sBase, 2u) << 16) | - 0xFF000000; - - Unsafe.As(ref dBase) = packed; - - sBase = ref Unsafe.Add(ref sBase, 3); - dBase = ref Unsafe.Add(ref dBase, 4); - } + // Preserve opaque W and Y while exchanging X and Z. + uint wy = source & 0xFF00FF00; + uint xz = source & 0x00FF00FF; + return wy | BitOperations.RotateLeft(xz, 16); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs index 3907df58c..eab5a41da 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs @@ -1,51 +1,37 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using static SixLabors.ImageSharp.SimdUtils; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp; -/// +/// +/// Identifies a stateless three-component shuffle operator. +/// internal interface IShuffle3 : IComponentShuffle { } -internal readonly struct DefaultShuffle3([ConstantExpected] byte control) : IShuffle3 +/// +/// Reorders XYZ components to ZYX. +/// +internal readonly struct ZYXShuffle3 : IShuffle3 { - public byte Control { get; } = control; - - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) -#pragma warning disable CA1857 // A constant is expected for the parameter - => HwIntrinsics.Shuffle3Reduce(ref source, ref destination, this.Control); -#pragma warning restore CA1857 // A constant is expected for the parameter - + /// [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + public static uint Invoke(uint source) { - ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(destination); - - SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0); - - for (nuint i = 0; i < (uint)source.Length; i += 3) - { - // The scalar remainder can run in-place after the vector body. Load - // the full 3-byte pixel into a register-sized value before stores so - // channel swaps cannot corrupt later reads from the same pixel. - uint packed = - Unsafe.Add(ref sBase, i + 0u) | - ((uint)Unsafe.Add(ref sBase, i + 1u) << 8) | - ((uint)Unsafe.Add(ref sBase, i + 2u) << 16); - - ref byte pBase = ref Unsafe.As(ref packed); - - Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0); - Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1); - Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2); - } + // Y is already centered; shift X and Z directly into each other's byte positions. + uint y = source & 0x0000FF00; + uint x = (source & 0x000000FF) << 16; + uint z = (source & 0x00FF0000) >> 16; + return x | y | z; } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs index 68f34efd7..56ea59df5 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs @@ -2,183 +2,320 @@ // Licensed under the Six Labors Split License. using System.Buffers.Binary; -using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using static SixLabors.ImageSharp.SimdUtils; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp; -/// +/// +/// Defines a stateless operation over one packed four-component pixel. +/// internal interface IShuffle4 : IComponentShuffle { + /// + /// Reorders the packed pixels in a 256-bit vector. + /// + /// The source pixels. + /// The reordered pixels. + static abstract Vector256 Invoke(Vector256 source); + + /// + /// Reorders the packed pixels in a 512-bit vector. + /// + /// The source pixels. + /// The reordered pixels. + static abstract Vector512 Invoke(Vector512 source); } -internal readonly struct DefaultShuffle4([ConstantExpected] byte control) : IShuffle4 +/// +/// Reorders XYZW components to WXYZ. +/// +internal readonly struct WXYZShuffle4 : IShuffle4 { - public byte Control { get; } = control; - + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) -#pragma warning disable CA1857 // A constant is expected for the parameter - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, this.Control); -#pragma warning restore CA1857 // A constant is expected for the parameter + public static uint Invoke(uint source) + { + // source = [W Z Y X] + // ROTL(8, source) = [Z Y X W] + return BitOperations.RotateLeft(source, 8); + } - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, CreateMask()); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 source) { - ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(destination); - - SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0); - - for (nuint i = 0; i < (uint)source.Length; i += 4) - { - // The generic path may be used with source and destination pointing - // at the same pixel. Load all channels first so subsequent stores - // index only staged bytes, matching the specialized uint shuffles. - uint packed = Unsafe.As(ref Unsafe.Add(ref sBase, i)); - ref byte pBase = ref Unsafe.As(ref packed); - - Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0); - Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1); - Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2); - Unsafe.Add(ref dBase, i + 3u) = Unsafe.Add(ref pBase, p3); - } + // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices. + Vector128 mask = CreateMask(); + return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); } -} -internal readonly struct WXYZShuffle4 : IShuffle4 -{ - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle2103); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 source) + => Vector512_.ShuffleNative(source, CreateMask512()); - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + /// + /// Creates the indices that rotate each XYZW pixel to WXYZ within one 128-bit lane. + /// + /// The pixel-local byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateMask() + => Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14); + + /// + /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. + /// + /// The absolute byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateMask512() { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // ROTL(8, packed) = [Z Y X W] - Unsafe.Add(ref dBase, i) = (packed << 8) | (packed >> 24); - } + // Native vpshufb ignores the lane offsets, while Vector512.Shuffle treats the indices as + // absolute. Encoding both meanings keeps the AVX-512 and managed fallback results identical. + return Vector512.Create( + 0x0605040702010003UL, + 0x0E0D0C0F0A09080BUL, + 0x1615141712111013UL, + 0x1E1D1C1F1A19181BUL, + 0x2625242722212023UL, + 0x2E2D2C2F2A29282BUL, + 0x3635343732313033UL, + 0x3E3D3C3F3A39383BUL).AsByte(); } } +/// +/// Reorders XYZW components to WZYX. +/// internal readonly struct WZYXShuffle4 : IShuffle4 { + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0123); + public static uint Invoke(uint source) + { + // Reversing the integer's endianness also reverses the four byte components. + return BinaryPrimitives.ReverseEndianness(source); + } - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, CreateMask()); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 source) { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // REVERSE(packedArgb) = [X Y Z W] - Unsafe.Add(ref dBase, i) = BinaryPrimitives.ReverseEndianness(packed); - } + Vector128 mask = CreateMask(); + return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 source) + => Vector512_.ShuffleNative(source, CreateMask512()); + + /// + /// Creates the indices that reverse each XYZW pixel to WZYX within one 128-bit lane. + /// + /// The pixel-local byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateMask() + => Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + + /// + /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. + /// + /// The absolute byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateMask512() + => Vector512.Create( + 0x0405060700010203UL, + 0x0C0D0E0F08090A0BUL, + 0x1415161710111213UL, + 0x1C1D1E1F18191A1BUL, + 0x2425262720212223UL, + 0x2C2D2E2F28292A2BUL, + 0x3435363730313233UL, + 0x3C3D3E3F38393A3BUL).AsByte(); } +/// +/// Reorders XYZW components to YZWX. +/// internal readonly struct YZWXShuffle4 : IShuffle4 { + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0321); + public static uint Invoke(uint source) + { + // source = [W Z Y X] + // ROTR(8, source) = [X W Z Y] + return BitOperations.RotateRight(source, 8); + } - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, CreateMask()); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 source) { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // ROTR(8, packedArgb) = [Y Z W X] - Unsafe.Add(ref dBase, i) = BitOperations.RotateRight(packed, 8); - } + Vector128 mask = CreateMask(); + return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 source) + => Vector512_.ShuffleNative(source, CreateMask512()); + + /// + /// Creates the indices that rotate each XYZW pixel to YZWX within one 128-bit lane. + /// + /// The pixel-local byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateMask() + => Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12); + + /// + /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. + /// + /// The absolute byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateMask512() + => Vector512.Create( + 0x0407060500030201UL, + 0x0C0F0E0D080B0A09UL, + 0x1417161510131211UL, + 0x1C1F1E1D181B1A19UL, + 0x2427262520232221UL, + 0x2C2F2E2D282B2A29UL, + 0x3437363530333231UL, + 0x3C3F3E3D383B3A39UL).AsByte(); } +/// +/// Reorders XYZW components to ZYXW. +/// internal readonly struct ZYXWShuffle4 : IShuffle4 { + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3012); + public static uint Invoke(uint source) + { + // Preserve W and Y while rotating the masked X/Z bytes into each other's positions. + uint wy = source & 0xFF00FF00; + uint xz = source & 0x00FF00FF; + return wy | BitOperations.RotateLeft(xz, 16); + } - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, CreateMask()); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 source) { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // tmp1 = [W 0 Y 0] - // tmp2 = [0 Z 0 X] - // tmp3=ROTL(16, tmp2) = [0 X 0 Z] - // tmp1 + tmp3 = [W X Y Z] - uint tmp1 = packed & 0xFF00FF00; - uint tmp2 = packed & 0x00FF00FF; - uint tmp3 = BitOperations.RotateLeft(tmp2, 16); - - Unsafe.Add(ref dBase, i) = tmp1 + tmp3; - } + Vector128 mask = CreateMask(); + return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 source) + => Vector512_.ShuffleNative(source, CreateMask512()); + + /// + /// Creates the indices that exchange X and Z in each XYZW pixel within one 128-bit lane. + /// + /// The pixel-local byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateMask() + => Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15); + + /// + /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. + /// + /// The absolute byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateMask512() + => Vector512.Create( + 0x0704050603000102UL, + 0x0F0C0D0E0B08090AUL, + 0x1714151613101112UL, + 0x1F1C1D1E1B18191AUL, + 0x2724252623202122UL, + 0x2F2C2D2E2B28292AUL, + 0x3734353633303132UL, + 0x3F3C3D3E3B38393AUL).AsByte(); } +/// +/// Reorders XYZW components to XWZY. +/// internal readonly struct XWZYShuffle4 : IShuffle4 { + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle1230); + public static uint Invoke(uint source) + { + // Preserve X and Z while rotating the masked Y/W bytes into each other's positions. + uint xz = source & 0x00FF00FF; + uint yw = source & 0xFF00FF00; + return xz | BitOperations.RotateLeft(yw, 16); + } - [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, CreateMask()); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 source) { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref uint dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - uint n = (uint)source.Length / 4; - - for (nuint i = 0; i < n; i++) - { - uint packed = Unsafe.Add(ref sBase, i); - - // packed = [W Z Y X] - // tmp1 = [0 Z 0 X] - // tmp2 = [W 0 Y 0] - // tmp3=ROTL(16, tmp2) = [Y 0 W 0] - // tmp1 + tmp3 = [Y Z W X] - uint tmp1 = packed & 0x00FF00FF; - uint tmp2 = packed & 0xFF00FF00; - uint tmp3 = BitOperations.RotateLeft(tmp2, 16); - - Unsafe.Add(ref dBase, i) = tmp1 + tmp3; - } + Vector128 mask = CreateMask(); + return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 source) + => Vector512_.ShuffleNative(source, CreateMask512()); + + /// + /// Creates the indices that exchange Y and W in each XYZW pixel within one 128-bit lane. + /// + /// The pixel-local byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateMask() + => Vector128.Create((byte)0, 3, 2, 1, 4, 7, 6, 5, 8, 11, 10, 9, 12, 15, 14, 13); + + /// + /// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. + /// + /// The absolute byte shuffle indices. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateMask512() + => Vector512.Create( + 0x0506070401020300UL, + 0x0D0E0F0C090A0B08UL, + 0x1516171411121310UL, + 0x1D1E1F1C191A1B18UL, + 0x2526272421222320UL, + 0x2D2E2F2C292A2B28UL, + 0x3536373431323330UL, + 0x3D3E3F3C393A3B38UL).AsByte(); } diff --git a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs index 613406167..17ec503e5 100644 --- a/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs +++ b/src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs @@ -1,101 +1,85 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Diagnostics.CodeAnalysis; +using System.Buffers.Binary; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using static SixLabors.ImageSharp.SimdUtils; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp; -/// +/// +/// Defines a stateless operation that reorders four packed components before retaining three. +/// internal interface IShuffle4Slice3 : IComponentShuffle { } -internal readonly struct DefaultShuffle4Slice3([ConstantExpected] byte control) : IShuffle4Slice3 +/// +/// Preserves XYZ order and discards W. +/// +internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3 { - public byte Control { get; } = control; - - [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) -#pragma warning disable CA1857 // A constant is expected for the parameter - => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, this.Control); -#pragma warning restore CA1857 // A constant is expected for the parameter - + /// [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) - { - ref byte sBase = ref MemoryMarshal.GetReference(source); - ref byte dBase = ref MemoryMarshal.GetReference(destination); + public static uint Invoke(uint source) => source; - SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) => source; +} - for (nuint i = 0, j = 0; i < (uint)destination.Length; i += 3, j += 4) - { - // Shrinking 4-byte pixels to 3 bytes can still be called in-place by - // tail code. Read the complete source pixel first, then write only - // the requested channels into the destination triplet. - uint packed = Unsafe.As(ref Unsafe.Add(ref sBase, j)); - ref byte pBase = ref Unsafe.As(ref packed); +/// +/// Reorders XYZW components to YZW before discarding X. +/// +internal readonly struct YZWXShuffle4Slice3 : IShuffle4Slice3 +{ + /// + [MethodImpl(InliningOptions.ShortMethod)] + public static uint Invoke(uint source) => BitOperations.RotateRight(source, 8); - Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0); - Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1); - Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2); - } - } + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12)); } -internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3 +/// +/// Reorders XYZW components to WZY before discarding X. +/// +internal readonly struct WZYXShuffle4Slice3 : IShuffle4Slice3 { + /// [MethodImpl(InliningOptions.ShortMethod)] - public void ShuffleReduce(ref ReadOnlySpan source, ref Span destination) - => HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210); + public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); +} +/// +/// Reorders XYZW components to ZYX before discarding W. +/// +internal readonly struct ZYXWShuffle4Slice3 : IShuffle4Slice3 +{ + /// [MethodImpl(InliningOptions.ShortMethod)] - public void Shuffle(ReadOnlySpan source, Span destination) + public static uint Invoke(uint source) { - ref uint sBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); - ref Byte3 dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - - nint n = (nint)(uint)source.Length / 4; - nint m = Numerics.Modulo4(n); - nint u = n - m; - - ref uint sLoopEnd = ref Unsafe.Add(ref sBase, u); - ref uint sEnd = ref Unsafe.Add(ref sBase, n); - - while (Unsafe.IsAddressLessThan(ref sBase, ref sLoopEnd)) - { - // Stage the four source pixels before the 3-byte stores. Even - // though this path preserves XYZ order, the packed loads must happen - // before destination writes when the spans overlap. - uint packed0 = Unsafe.Add(ref sBase, 0u); - uint packed1 = Unsafe.Add(ref sBase, 1u); - uint packed2 = Unsafe.Add(ref sBase, 2u); - uint packed3 = Unsafe.Add(ref sBase, 3u); - - Unsafe.Add(ref dBase, 0u) = Unsafe.As(ref packed0); - Unsafe.Add(ref dBase, 1u) = Unsafe.As(ref packed1); - Unsafe.Add(ref dBase, 2u) = Unsafe.As(ref packed2); - Unsafe.Add(ref dBase, 3u) = Unsafe.As(ref packed3); - - sBase = ref Unsafe.Add(ref sBase, 4); - dBase = ref Unsafe.Add(ref dBase, 4); - } - - while (Unsafe.IsAddressLessThan(ref sBase, ref sEnd)) - { - // Same overlap rule as the unrolled loop: take the 4-byte source - // pixel before storing the 3-byte destination value. - uint packed = Unsafe.Add(ref sBase, 0u); - - Unsafe.Add(ref dBase, 0u) = Unsafe.As(ref packed); - - sBase = ref Unsafe.Add(ref sBase, 1); - dBase = ref Unsafe.Add(ref dBase, 1); - } + // Preserve W and Y while exchanging X and Z; W is subsequently discarded. + uint wy = source & 0xFF00FF00; + uint xz = source & 0x00FF00FF; + return wy | BitOperations.RotateLeft(xz, 16); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 source) + => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); } [StructLayout(LayoutKind.Explicit, Size = 3)] diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs b/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs index 8b2baec21..e709cc8d4 100644 --- a/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs +++ b/src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs @@ -6,6 +6,8 @@ using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp; @@ -42,22 +44,104 @@ internal static partial class SimdUtils /// The type of shuffle struct. /// The source span of bytes. /// The destination span of bytes. - /// The type of shuffle to perform. [MethodImpl(InliningOptions.ShortMethod)] public static void Shuffle4( ReadOnlySpan source, - Span destination, - TShuffle shuffle) + Span destination) where TShuffle : struct, IShuffle4 { VerifyShuffle4SpanInput(source, destination); - shuffle.ShuffleReduce(ref source, ref destination); + ref byte sourceBase = ref MemoryMarshal.GetReference(source); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + int length = source.Length; + int i = 0; - // Deal with the remainder: - if (source.Length > 0) + // The same offset flows through descending widths. This keeps a single traversal while + // allowing a row that is not a multiple of the widest register to retain a vectorized tail. + if (Vector512.IsHardwareAccelerated) + { + int fourVectorsFromEnd = length - (Vector512.Count * 4); + + for (; i <= fourVectorsFromEnd; i += Vector512.Count * 4) + { + // Four independent vectors amortize loop control and expose enough work for the CPU + // to overlap loads, byte shuffles, and stores without changing pixel ordering. + TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i)) + .StoreUnsafe(ref destinationBase, (nuint)i); + TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + Vector512.Count))) + .StoreUnsafe(ref destinationBase, (nuint)(i + Vector512.Count)); + TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512.Count * 2)))) + .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512.Count * 2))); + TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512.Count * 3)))) + .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512.Count * 3))); + } + + int oneVectorFromEnd = length - Vector512.Count; + + for (; i <= oneVectorFromEnd; i += Vector512.Count) + { + TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i)) + .StoreUnsafe(ref destinationBase, (nuint)i); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int fourVectorsFromEnd = length - (Vector256.Count * 4); + + for (; i <= fourVectorsFromEnd; i += Vector256.Count * 4) + { + TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i)) + .StoreUnsafe(ref destinationBase, (nuint)i); + TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + Vector256.Count))) + .StoreUnsafe(ref destinationBase, (nuint)(i + Vector256.Count)); + TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256.Count * 2)))) + .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256.Count * 2))); + TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256.Count * 3)))) + .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256.Count * 3))); + } + + int oneVectorFromEnd = length - Vector256.Count; + + for (; i <= oneVectorFromEnd; i += Vector256.Count) + { + TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i)) + .StoreUnsafe(ref destinationBase, (nuint)i); + } + } + + if (Vector128.IsHardwareAccelerated) { - shuffle.Shuffle(source, destination); + int fourVectorsFromEnd = length - (Vector128.Count * 4); + + for (; i <= fourVectorsFromEnd; i += Vector128.Count * 4) + { + TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i)) + .StoreUnsafe(ref destinationBase, (nuint)i); + TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + Vector128.Count))) + .StoreUnsafe(ref destinationBase, (nuint)(i + Vector128.Count)); + TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128.Count * 2)))) + .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128.Count * 2))); + TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128.Count * 3)))) + .StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128.Count * 3))); + } + + int oneVectorFromEnd = length - Vector128.Count; + + for (; i <= oneVectorFromEnd; i += Vector128.Count) + { + TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i)) + .StoreUnsafe(ref destinationBase, (nuint)i); + } + } + + // The vector cascade leaves fewer than four pixels. A full uint load keeps each pixel + // in a register while the closed operator resolves to its rotate, reverse, or mask sequence. + for (; i < length; i += 4) + { + uint packed = Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)i)); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)i)) = TShuffle.Invoke(packed); } } @@ -68,23 +152,112 @@ internal static partial class SimdUtils /// The type of shuffle struct. /// The source span of bytes. /// The destination span of bytes. - /// The type of shuffle to perform. [MethodImpl(InliningOptions.ShortMethod)] public static void Shuffle3( ReadOnlySpan source, - Span destination, - TShuffle shuffle) + Span destination) where TShuffle : struct, IShuffle3 { - // Source length should be smaller than destination length, and divisible by 3. VerifyShuffle3SpanInput(source, destination); - shuffle.ShuffleReduce(ref source, ref destination); + ref byte sourceBase = ref MemoryMarshal.GetReference(source); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + int length = source.Length; + int i = 0; - // Deal with the remainder: - if (source.Length > 0) + if (Vector128.IsHardwareAccelerated) + { + // Each group contains sixteen XYZ pixels in three registers. The pad mask expands + // four triplets per register to XYZW, with 0x80 selecting zero for the temporary W lane. + Vector128 padMask = Vector128.Create( + (byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80); + + // After the operator has reordered padded pixels, these masks remove every temporary + // W lane and repack the four registers into three contiguous XYZ destination registers. + Vector128 sliceMask = Vector128.Create( + (byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80); + Vector128 sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12); + ref Vector128 sourceVectors = ref Unsafe.As>(ref sourceBase); + ref Vector128 destinationVectors = ref Unsafe.As>(ref destinationBase); + nuint sourceVectorCount = (uint)length / (uint)Vector128.Count; + nuint vectorIndex = 0; + + for (; vectorIndex + 2 < sourceVectorCount; vectorIndex += 3) + { + // Realign the three source registers into four registers holding four complete + // triplets apiece. All source registers are captured before any destination store. + ref Vector128 source0 = ref Unsafe.Add(ref sourceVectors, vectorIndex); + Vector128 v0 = source0; + Vector128 v1 = Unsafe.Add(ref source0, 1); + Vector128 v2 = Unsafe.Add(ref source0, 2); + Vector128 v3 = Vector128_.ShiftRightBytesInVector(v2, 4); + + v2 = Vector128_.AlignRight(v2, v1, 8); + v1 = Vector128_.AlignRight(v1, v0, 12); + + v0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask)); + v1 = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask)); + v2 = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask)); + v3 = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask)); + + v0 = Vector128_.ShuffleNative(v0, sliceEndMask); + v1 = Vector128_.ShuffleNative(v1, sliceMask); + v2 = Vector128_.ShuffleNative(v2, sliceEndMask); + v3 = Vector128_.ShuffleNative(v3, sliceMask); + + Vector128 destination0 = Vector128_.AlignRight(v1, v0, 4); + Vector128 destination2 = Vector128_.AlignRight(v3, v2, 12); + v1 = Vector128_.ShiftLeftBytesInVector(v1, 4); + v2 = Vector128_.ShiftRightBytesInVector(v2, 4); + Vector128 destination1 = Vector128_.AlignRight(v2, v1, 8); + + ref Vector128 destination0Ref = ref Unsafe.Add(ref destinationVectors, vectorIndex); + destination0Ref = destination0; + Unsafe.Add(ref destination0Ref, 1) = destination1; + Unsafe.Add(ref destination0Ref, 2) = destination2; + } + + i = (int)(vectorIndex * (uint)Vector128.Count); + int oneTailVectorFromEnd = length - Vector128.Count; + + for (; i <= oneTailVectorFromEnd; i += 12) + { + // A single readable register contains four complete triplets plus four bytes from + // the following pixels. The pad mask ignores those extra bytes before the operator + // runs, and the slice mask packs the four results into the low twelve bytes. + Vector128 result = Vector128.LoadUnsafe(ref sourceBase, (nuint)i); + result = Vector128_.ShuffleNative(result, padMask); + result = TShuffle.Invoke(result); + result = Vector128_.ShuffleNative(result, sliceMask); + + // Store exactly twelve bytes so an in-place shuffle does not overwrite the next + // source triplet captured by the following iteration. + Unsafe.As>(ref Unsafe.Add(ref destinationBase, (nuint)i)) = result.GetLower(); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)(i + 8))) = result.AsUInt32().GetElement(2); + } + } + + int widenedReadEnd = length - 3; + + for (; i < widenedReadEnd; i += 3) { - shuffle.Shuffle(source, destination); + // The fourth byte belongs to the following pixel, but the operator only contributes the + // low three result bytes. This unaligned read replaces three dependent byte loads safely. + uint packed = Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)i)); + uint shuffled = TShuffle.Invoke(packed); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)i)) = Unsafe.As(ref shuffled); + } + + if (i < length) + { + // The final triplet has no fourth readable byte, so construct only this terminal pixel. + uint packed = + Unsafe.Add(ref sourceBase, (nuint)i) | + ((uint)Unsafe.Add(ref sourceBase, (nuint)(i + 1)) << 8) | + ((uint)Unsafe.Add(ref sourceBase, (nuint)(i + 2)) << 16); + + uint shuffled = TShuffle.Invoke(packed); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)i)) = Unsafe.As(ref shuffled); } } @@ -95,22 +268,78 @@ internal static partial class SimdUtils /// The type of shuffle struct. /// The source span of bytes. /// The destination span of bytes. - /// The type of shuffle to perform. [MethodImpl(InliningOptions.ShortMethod)] public static void Pad3Shuffle4( ReadOnlySpan source, - Span destination, - TShuffle shuffle) + Span destination) where TShuffle : struct, IPad3Shuffle4 { VerifyPad3Shuffle4SpanInput(source, destination); - shuffle.ShuffleReduce(ref source, ref destination); + ref byte sourceBase = ref MemoryMarshal.GetReference(source); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + int sourceLength = source.Length; + int sourceOffset = 0; + int destinationOffset = 0; - // Deal with the remainder: - if (source.Length > 0) + if (Vector128.IsHardwareAccelerated) + { + // The fixed mask expands four XYZ triplets to four XYZW pixels. The zeroed W bytes + // are then filled with opaque alpha before the selected operator reorders each pixel. + Vector128 padMask = Vector128.Create( + (byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80); + Vector128 opaqueAlpha = Vector128.Create(0xFF000000FF000000UL).AsByte(); + ref Vector128 sourceVectors = ref Unsafe.As>(ref sourceBase); + ref Vector128 destinationVectors = ref Unsafe.As>(ref destinationBase); + nuint sourceVectorCount = (uint)sourceLength / (uint)Vector128.Count; + nuint sourceVectorIndex = 0; + nuint destinationVectorIndex = 0; + + for (; sourceVectorIndex + 2 < sourceVectorCount; + sourceVectorIndex += 3, destinationVectorIndex += 4) + { + // Three source registers contain sixteen packed triplets. Aligning at 12, 8, and + // 4-byte boundaries produces four registers whose low twelve bytes each hold four pixels. + ref Vector128 source0 = ref Unsafe.Add(ref sourceVectors, sourceVectorIndex); + Vector128 v0 = source0; + Vector128 v1 = Unsafe.Add(ref source0, 1); + Vector128 v2 = Unsafe.Add(ref source0, 2); + Vector128 v3 = Vector128_.ShiftRightBytesInVector(v2, 4); + + v2 = Vector128_.AlignRight(v2, v1, 8); + v1 = Vector128_.AlignRight(v1, v0, 12); + + ref Vector128 destination0 = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex); + destination0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask) | opaqueAlpha); + Unsafe.Add(ref destination0, 1) = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask) | opaqueAlpha); + Unsafe.Add(ref destination0, 2) = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask) | opaqueAlpha); + Unsafe.Add(ref destination0, 3) = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask) | opaqueAlpha); + } + + sourceOffset = (int)(sourceVectorIndex * (uint)Vector128.Count); + destinationOffset = (int)(destinationVectorIndex * (uint)Vector128.Count); + } + + int widenedReadEnd = sourceLength - 3; + + for (; sourceOffset < widenedReadEnd; sourceOffset += 3, destinationOffset += 4) + { + // The widened load intentionally includes the next pixel's first byte. Replacing that + // high byte with opaque alpha yields the complete XYZW value with one unaligned read. + uint packed = Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset)) | 0xFF000000; + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = TShuffle.Invoke(packed); + } + + if (sourceOffset < sourceLength) { - shuffle.Shuffle(source, destination); + // The final triplet cannot use the widened load because no following byte is in range. + uint packed = + Unsafe.Add(ref sourceBase, (nuint)sourceOffset) | + ((uint)Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 1)) << 8) | + ((uint)Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 2)) << 16) | + 0xFF000000; + + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = TShuffle.Invoke(packed); } } @@ -121,22 +350,101 @@ internal static partial class SimdUtils /// The type of shuffle struct. /// The source span of bytes. /// The destination span of bytes. - /// The type of shuffle to perform. [MethodImpl(InliningOptions.ShortMethod)] public static void Shuffle4Slice3( ReadOnlySpan source, - Span destination, - TShuffle shuffle) + Span destination) where TShuffle : struct, IShuffle4Slice3 { VerifyShuffle4Slice3SpanInput(source, destination); - shuffle.ShuffleReduce(ref source, ref destination); + ref byte sourceBase = ref MemoryMarshal.GetReference(source); + ref byte destinationBase = ref MemoryMarshal.GetReference(destination); + int sourceLength = source.Length; + int sourceOffset = 0; + int destinationOffset = 0; - // Deal with the remainder: - if (source.Length > 0) + if (Vector128.IsHardwareAccelerated) + { + // Each operator first places the three retained components in the low bytes of every + // four-byte pixel. These masks then delete the fourth byte and compact sixteen pixels. + Vector128 sliceMask = Vector128.Create( + (byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80); + Vector128 sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12); + ref Vector128 sourceVectors = ref Unsafe.As>(ref sourceBase); + ref Vector128 destinationVectors = ref Unsafe.As>(ref destinationBase); + nuint sourceVectorCount = (uint)sourceLength / (uint)Vector128.Count; + nuint sourceVectorIndex = 0; + nuint destinationVectorIndex = 0; + + for (; sourceVectorIndex + 3 < sourceVectorCount; + sourceVectorIndex += 4, destinationVectorIndex += 3) + { + // Load and transform all sixteen source pixels before writing the shorter output group. + // This preserves forward progress when source and destination begin at the same address. + ref Vector128 source0 = ref Unsafe.Add(ref sourceVectors, sourceVectorIndex); + Vector128 v0 = TShuffle.Invoke(source0); + Vector128 v1 = TShuffle.Invoke(Unsafe.Add(ref source0, 1)); + Vector128 v2 = TShuffle.Invoke(Unsafe.Add(ref source0, 2)); + Vector128 v3 = TShuffle.Invoke(Unsafe.Add(ref source0, 3)); + + v0 = Vector128_.ShuffleNative(v0, sliceEndMask); + v1 = Vector128_.ShuffleNative(v1, sliceMask); + v2 = Vector128_.ShuffleNative(v2, sliceEndMask); + v3 = Vector128_.ShuffleNative(v3, sliceMask); + + Vector128 destination0 = Vector128_.AlignRight(v1, v0, 4); + Vector128 destination2 = Vector128_.AlignRight(v3, v2, 12); + v1 = Vector128_.ShiftLeftBytesInVector(v1, 4); + v2 = Vector128_.ShiftRightBytesInVector(v2, 4); + Vector128 destination1 = Vector128_.AlignRight(v2, v1, 8); + + ref Vector128 destination0Ref = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex); + destination0Ref = destination0; + Unsafe.Add(ref destination0Ref, 1) = destination1; + Unsafe.Add(ref destination0Ref, 2) = destination2; + } + + sourceOffset = (int)(sourceVectorIndex * (uint)Vector128.Count); + destinationOffset = (int)(destinationVectorIndex * (uint)Vector128.Count); + int oneTailVectorFromEnd = sourceLength - Vector128.Count; + + for (; sourceOffset <= oneTailVectorFromEnd; sourceOffset += 16, destinationOffset += 12) + { + // The operator arranges the three retained components at the front of each pixel. + // One fixed shuffle then compacts four pixels into the low twelve vector bytes. + Vector128 result = TShuffle.Invoke( + Vector128.LoadUnsafe(ref sourceBase, (nuint)sourceOffset)); + + result = Vector128_.ShuffleNative(result, sliceMask); + + // The split store writes the exact 12-byte result and remains safe for in-place shrinking. + Unsafe.As>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = result.GetLower(); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 8))) = result.AsUInt32().GetElement(2); + } + } + + int fourPixelsFromEnd = sourceLength - 16; + + for (; sourceOffset <= fourPixelsFromEnd; sourceOffset += 16, destinationOffset += 12) + { + // Transform four complete pixels before the first three-byte store. Keeping the source + // values in registers avoids reloads after an in-place shrinking destination advances. + uint packed0 = TShuffle.Invoke(Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset))); + uint packed1 = TShuffle.Invoke(Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 4)))); + uint packed2 = TShuffle.Invoke(Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 8)))); + uint packed3 = TShuffle.Invoke(Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 12)))); + + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = Unsafe.As(ref packed0); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 3))) = Unsafe.As(ref packed1); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 6))) = Unsafe.As(ref packed2); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 9))) = Unsafe.As(ref packed3); + } + + for (; sourceOffset < sourceLength; sourceOffset += 4, destinationOffset += 3) { - shuffle.Shuffle(source, destination); + uint packed = TShuffle.Invoke(Unsafe.As(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset))); + Unsafe.As(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = Unsafe.As(ref packed); } } diff --git a/src/ImageSharp/PixelFormats/Utils/PixelConverter.cs b/src/ImageSharp/PixelFormats/Utils/PixelConverter.cs index 50a68906e..c25cc301c 100644 --- a/src/ImageSharp/PixelFormats/Utils/PixelConverter.cs +++ b/src/ImageSharp/PixelFormats/Utils/PixelConverter.cs @@ -33,7 +33,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToArgb32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -44,7 +44,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgra32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -55,7 +55,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToAbgr32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -66,7 +66,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgb24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, default); + => SimdUtils.Shuffle4Slice3(source, dest); /// /// Converts a representing a collection of @@ -77,7 +77,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgr24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle3012)); + => SimdUtils.Shuffle4Slice3(source, dest); } /// @@ -96,7 +96,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgba32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -107,7 +107,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgra32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -118,7 +118,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToAbgr32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -129,7 +129,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgb24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0321)); + => SimdUtils.Shuffle4Slice3(source, dest); /// /// Converts a representing a collection of @@ -140,7 +140,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgr24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0123)); + => SimdUtils.Shuffle4Slice3(source, dest); } /// @@ -159,7 +159,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToArgb32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -170,7 +170,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgba32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -181,7 +181,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToAbgr32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -192,7 +192,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgb24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle3012)); + => SimdUtils.Shuffle4Slice3(source, dest); /// /// Converts a representing a collection of @@ -203,7 +203,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgr24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, default); + => SimdUtils.Shuffle4Slice3(source, dest); } /// @@ -222,7 +222,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToArgb32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -233,7 +233,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgba32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -244,7 +244,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgra32(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4(source, dest, default); + => SimdUtils.Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -255,7 +255,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgb24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0123)); + => SimdUtils.Shuffle4Slice3(source, dest); /// /// Converts a representing a collection of @@ -266,7 +266,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgr24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0321)); + => SimdUtils.Shuffle4Slice3(source, dest); } /// @@ -285,7 +285,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgba32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, default); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -296,7 +296,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToArgb32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle2103)); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -307,7 +307,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgra32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle3012)); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -318,7 +318,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToAbgr32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle0123)); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -329,7 +329,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgr24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle3(source, dest, new DefaultShuffle3(SimdUtils.Shuffle.MMShuffle3012)); + => SimdUtils.Shuffle3(source, dest); } /// @@ -348,7 +348,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToArgb32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle0123)); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -359,7 +359,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgba32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle3012)); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -370,7 +370,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToBgra32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, default); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -381,7 +381,7 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToAbgr32(ReadOnlySpan source, Span dest) - => SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle2103)); + => SimdUtils.Pad3Shuffle4(source, dest); /// /// Converts a representing a collection of @@ -392,6 +392,6 @@ internal static class PixelConverter /// The destination span of bytes. [MethodImpl(InliningOptions.ShortMethod)] public static void ToRgb24(ReadOnlySpan source, Span dest) - => SimdUtils.Shuffle3(source, dest, new DefaultShuffle3(SimdUtils.Shuffle.MMShuffle3012)); + => SimdUtils.Shuffle3(source, dest); } } diff --git a/tests/ImageSharp.Benchmarks/Bulk/Pad3Shuffle4Channel.cs b/tests/ImageSharp.Benchmarks/Bulk/Pad3Shuffle4Channel.cs index 8728dd671..16529306a 100644 --- a/tests/ImageSharp.Benchmarks/Bulk/Pad3Shuffle4Channel.cs +++ b/tests/ImageSharp.Benchmarks/Bulk/Pad3Shuffle4Channel.cs @@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Benchmarks.Bulk; [Config(typeof(Config.HwIntrinsics_SSE_AVX))] public class Pad3Shuffle4Channel { - private static readonly DefaultPad3Shuffle4 Control = new(SimdUtils.Shuffle.MMShuffle1032); private byte[] source; private byte[] destination; @@ -25,11 +24,11 @@ public class Pad3Shuffle4Channel [Benchmark] public void Pad3Shuffle4() - => SimdUtils.Pad3Shuffle4(this.source, this.destination, Control); + => SimdUtils.Pad3Shuffle4(this.source, this.destination); [Benchmark] public void Pad3Shuffle4FastFallback() - => SimdUtils.Pad3Shuffle4(this.source, this.destination, default(XYZWPad3Shuffle4)); + => SimdUtils.Pad3Shuffle4(this.source, this.destination); } // 2020-10-30 diff --git a/tests/ImageSharp.Benchmarks/Bulk/Shuffle3Channel.cs b/tests/ImageSharp.Benchmarks/Bulk/Shuffle3Channel.cs index e4c12900f..e09c4ce6c 100644 --- a/tests/ImageSharp.Benchmarks/Bulk/Shuffle3Channel.cs +++ b/tests/ImageSharp.Benchmarks/Bulk/Shuffle3Channel.cs @@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Benchmarks.Bulk; [Config(typeof(Config.HwIntrinsics_SSE_AVX))] public class Shuffle3Channel { - private static readonly DefaultShuffle3 Control = new(SimdUtils.Shuffle.MMShuffle3102); private byte[] source; private byte[] destination; @@ -25,7 +24,7 @@ public class Shuffle3Channel [Benchmark] public void Shuffle3() - => SimdUtils.Shuffle3(this.source, this.destination, Control); + => SimdUtils.Shuffle3(this.source, this.destination); } // 2020-11-02 diff --git a/tests/ImageSharp.Benchmarks/Bulk/Shuffle4Slice3Channel.cs b/tests/ImageSharp.Benchmarks/Bulk/Shuffle4Slice3Channel.cs index 579e2c54d..4b694768d 100644 --- a/tests/ImageSharp.Benchmarks/Bulk/Shuffle4Slice3Channel.cs +++ b/tests/ImageSharp.Benchmarks/Bulk/Shuffle4Slice3Channel.cs @@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Benchmarks.Bulk; [Config(typeof(Config.HwIntrinsics_SSE_AVX))] public class Shuffle4Slice3Channel { - private static readonly DefaultShuffle4Slice3 Control = new(SimdUtils.Shuffle.MMShuffle1032); private byte[] source; private byte[] destination; @@ -25,11 +24,11 @@ public class Shuffle4Slice3Channel [Benchmark] public void Shuffle4Slice3() - => SimdUtils.Shuffle4Slice3(this.source, this.destination, Control); + => SimdUtils.Shuffle4Slice3(this.source, this.destination); [Benchmark] public void Shuffle4Slice3FastFallback() - => SimdUtils.Shuffle4Slice3(this.source, this.destination, default(XYZWShuffle4Slice3)); + => SimdUtils.Shuffle4Slice3(this.source, this.destination); } // 2020-10-29 diff --git a/tests/ImageSharp.Benchmarks/Bulk/ShuffleByte4Channel.cs b/tests/ImageSharp.Benchmarks/Bulk/ShuffleByte4Channel.cs index 6a16bb571..f3ad974c4 100644 --- a/tests/ImageSharp.Benchmarks/Bulk/ShuffleByte4Channel.cs +++ b/tests/ImageSharp.Benchmarks/Bulk/ShuffleByte4Channel.cs @@ -24,7 +24,7 @@ public class ShuffleByte4Channel [Benchmark] public void Shuffle4Channel() - => SimdUtils.Shuffle4(this.source, this.destination, default); + => SimdUtils.Shuffle4(this.source, this.destination); } // 2020-10-29 diff --git a/tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversion.cs b/tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversion.cs new file mode 100644 index 000000000..a7dcb9021 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversion.cs @@ -0,0 +1,222 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.PixelFormats.Utils; + +namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion; + +/// +/// Measures every optimized conversion between the six byte-packed RGB and RGBA pixel layouts. +/// +[Config(typeof(Config.Short))] +public class PackedPixelConversion +{ + private byte[] source3; + private byte[] source4; + private byte[] destination3; + private byte[] destination4; + + /// + /// Gets or sets the number of pixels converted by each invocation. + /// + [Params(7, 256, 4096)] + public int Count { get; set; } + + /// + /// Creates deterministic source buffers and correctly sized destination buffers. + /// + [GlobalSetup] + public void Setup() + { + this.source3 = new byte[this.Count * 3]; + this.source4 = new byte[this.Count * 4]; + this.destination3 = new byte[this.Count * 3]; + this.destination4 = new byte[this.Count * 4]; + + // Non-repeating channel values prevent the JIT or hardware from benefiting from + // zero-filled inputs while keeping both benchmark revisions byte-for-byte identical. + new Random(42).NextBytes(this.source3); + new Random(42).NextBytes(this.source4); + } + + /// + /// Converts RGBA pixels to ARGB pixels. + /// + [Benchmark] + public void Rgba32ToArgb32() => PixelConverter.FromRgba32.ToArgb32(this.source4, this.destination4); + + /// + /// Converts RGBA pixels to ABGR pixels. + /// + [Benchmark] + public void Rgba32ToAbgr32() => PixelConverter.FromRgba32.ToAbgr32(this.source4, this.destination4); + + /// + /// Converts RGBA pixels to BGRA pixels. + /// + [Benchmark] + public void Rgba32ToBgra32() => PixelConverter.FromRgba32.ToBgra32(this.source4, this.destination4); + + /// + /// Converts RGBA pixels to RGB pixels. + /// + [Benchmark] + public void Rgba32ToRgb24() => PixelConverter.FromRgba32.ToRgb24(this.source4, this.destination3); + + /// + /// Converts RGBA pixels to BGR pixels. + /// + [Benchmark] + public void Rgba32ToBgr24() => PixelConverter.FromRgba32.ToBgr24(this.source4, this.destination3); + + /// + /// Converts ARGB pixels to RGBA pixels. + /// + [Benchmark] + public void Argb32ToRgba32() => PixelConverter.FromArgb32.ToRgba32(this.source4, this.destination4); + + /// + /// Converts ARGB pixels to ABGR pixels. + /// + [Benchmark] + public void Argb32ToAbgr32() => PixelConverter.FromArgb32.ToAbgr32(this.source4, this.destination4); + + /// + /// Converts ARGB pixels to BGRA pixels. + /// + [Benchmark] + public void Argb32ToBgra32() => PixelConverter.FromArgb32.ToBgra32(this.source4, this.destination4); + + /// + /// Converts ARGB pixels to RGB pixels. + /// + [Benchmark] + public void Argb32ToRgb24() => PixelConverter.FromArgb32.ToRgb24(this.source4, this.destination3); + + /// + /// Converts ARGB pixels to BGR pixels. + /// + [Benchmark] + public void Argb32ToBgr24() => PixelConverter.FromArgb32.ToBgr24(this.source4, this.destination3); + + /// + /// Converts ABGR pixels to RGBA pixels. + /// + [Benchmark] + public void Abgr32ToRgba32() => PixelConverter.FromAbgr32.ToRgba32(this.source4, this.destination4); + + /// + /// Converts ABGR pixels to ARGB pixels. + /// + [Benchmark] + public void Abgr32ToArgb32() => PixelConverter.FromAbgr32.ToArgb32(this.source4, this.destination4); + + /// + /// Converts ABGR pixels to BGRA pixels. + /// + [Benchmark] + public void Abgr32ToBgra32() => PixelConverter.FromAbgr32.ToBgra32(this.source4, this.destination4); + + /// + /// Converts ABGR pixels to RGB pixels. + /// + [Benchmark] + public void Abgr32ToRgb24() => PixelConverter.FromAbgr32.ToRgb24(this.source4, this.destination3); + + /// + /// Converts ABGR pixels to BGR pixels. + /// + [Benchmark] + public void Abgr32ToBgr24() => PixelConverter.FromAbgr32.ToBgr24(this.source4, this.destination3); + + /// + /// Converts BGRA pixels to RGBA pixels. + /// + [Benchmark] + public void Bgra32ToRgba32() => PixelConverter.FromBgra32.ToRgba32(this.source4, this.destination4); + + /// + /// Converts BGRA pixels to ARGB pixels. + /// + [Benchmark] + public void Bgra32ToArgb32() => PixelConverter.FromBgra32.ToArgb32(this.source4, this.destination4); + + /// + /// Converts BGRA pixels to ABGR pixels. + /// + [Benchmark] + public void Bgra32ToAbgr32() => PixelConverter.FromBgra32.ToAbgr32(this.source4, this.destination4); + + /// + /// Converts BGRA pixels to RGB pixels. + /// + [Benchmark] + public void Bgra32ToRgb24() => PixelConverter.FromBgra32.ToRgb24(this.source4, this.destination3); + + /// + /// Converts BGRA pixels to BGR pixels. + /// + [Benchmark] + public void Bgra32ToBgr24() => PixelConverter.FromBgra32.ToBgr24(this.source4, this.destination3); + + /// + /// Converts RGB pixels to RGBA pixels. + /// + [Benchmark] + public void Rgb24ToRgba32() => PixelConverter.FromRgb24.ToRgba32(this.source3, this.destination4); + + /// + /// Converts RGB pixels to ARGB pixels. + /// + [Benchmark] + public void Rgb24ToArgb32() => PixelConverter.FromRgb24.ToArgb32(this.source3, this.destination4); + + /// + /// Converts RGB pixels to ABGR pixels. + /// + [Benchmark] + public void Rgb24ToAbgr32() => PixelConverter.FromRgb24.ToAbgr32(this.source3, this.destination4); + + /// + /// Converts RGB pixels to BGRA pixels. + /// + [Benchmark] + public void Rgb24ToBgra32() => PixelConverter.FromRgb24.ToBgra32(this.source3, this.destination4); + + /// + /// Converts RGB pixels to BGR pixels. + /// + [Benchmark] + public void Rgb24ToBgr24() => PixelConverter.FromRgb24.ToBgr24(this.source3, this.destination3); + + /// + /// Converts BGR pixels to RGBA pixels. + /// + [Benchmark] + public void Bgr24ToRgba32() => PixelConverter.FromBgr24.ToRgba32(this.source3, this.destination4); + + /// + /// Converts BGR pixels to ARGB pixels. + /// + [Benchmark] + public void Bgr24ToArgb32() => PixelConverter.FromBgr24.ToArgb32(this.source3, this.destination4); + + /// + /// Converts BGR pixels to ABGR pixels. + /// + [Benchmark] + public void Bgr24ToAbgr32() => PixelConverter.FromBgr24.ToAbgr32(this.source3, this.destination4); + + /// + /// Converts BGR pixels to BGRA pixels. + /// + [Benchmark] + public void Bgr24ToBgra32() => PixelConverter.FromBgr24.ToBgra32(this.source3, this.destination4); + + /// + /// Converts BGR pixels to RGB pixels. + /// + [Benchmark] + public void Bgr24ToRgb24() => PixelConverter.FromBgr24.ToRgb24(this.source3, this.destination3); +} diff --git a/tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs b/tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs new file mode 100644 index 000000000..b93e40e19 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs @@ -0,0 +1,128 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; + +namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion; + +/// +/// Exposes every stateless packed-pixel shuffle operator for assembly inspection. +/// +/// +/// Seven pixels expose the short-input and vector-tail paths. Seventeen pixels execute one +/// full intrinsic group and leave one complete pixel for the scalar remainder, making every +/// part of each generated traversal observable. +/// +[Config(typeof(Config.Analysis))] +public class PackedPixelConversionAssembly +{ + private byte[] source3; + private byte[] source4; + private byte[] destination3; + private byte[] destination4; + + /// + /// Gets or sets the number of pixels converted by each invocation. + /// + [Params(7, 17)] + public int Count { get; set; } + + /// + /// Populates the source buffers with deterministic non-uniform channel values. + /// + [GlobalSetup] + public void Setup() + { + this.source3 = new byte[this.Count * 3]; + this.source4 = new byte[this.Count * 4]; + this.destination3 = new byte[this.Count * 3]; + this.destination4 = new byte[this.Count * 4]; + + new Random(42).NextBytes(this.source3); + new Random(42).NextBytes(this.source4); + } + + /// + /// Executes the WXYZ four-to-four operator. + /// + [Benchmark] + public void Shuffle4Wxyz() => SimdUtils.Shuffle4(this.source4, this.destination4); + + /// + /// Executes the WZYX four-to-four operator. + /// + [Benchmark] + public void Shuffle4Wzyx() => SimdUtils.Shuffle4(this.source4, this.destination4); + + /// + /// Executes the YZWX four-to-four operator. + /// + [Benchmark] + public void Shuffle4Yzwx() => SimdUtils.Shuffle4(this.source4, this.destination4); + + /// + /// Executes the ZYXW four-to-four operator. + /// + [Benchmark] + public void Shuffle4Zyxw() => SimdUtils.Shuffle4(this.source4, this.destination4); + + /// + /// Executes the XWZY four-to-four operator. + /// + [Benchmark] + public void Shuffle4Xwzy() => SimdUtils.Shuffle4(this.source4, this.destination4); + + /// + /// Executes the XYZ four-to-three operator. + /// + [Benchmark] + public void Slice3Xyz() => SimdUtils.Shuffle4Slice3(this.source4, this.destination3); + + /// + /// Executes the YZW four-to-three operator. + /// + [Benchmark] + public void Slice3Yzw() => SimdUtils.Shuffle4Slice3(this.source4, this.destination3); + + /// + /// Executes the WZY four-to-three operator. + /// + [Benchmark] + public void Slice3Wzy() => SimdUtils.Shuffle4Slice3(this.source4, this.destination3); + + /// + /// Executes the ZYX four-to-three operator. + /// + [Benchmark] + public void Slice3Zyx() => SimdUtils.Shuffle4Slice3(this.source4, this.destination3); + + /// + /// Executes the XYZW three-to-four operator. + /// + [Benchmark] + public void Pad4Xyzw() => SimdUtils.Pad3Shuffle4(this.source3, this.destination4); + + /// + /// Executes the WXYZ three-to-four operator. + /// + [Benchmark] + public void Pad4Wxyz() => SimdUtils.Pad3Shuffle4(this.source3, this.destination4); + + /// + /// Executes the WZYX three-to-four operator. + /// + [Benchmark] + public void Pad4Wzyx() => SimdUtils.Pad3Shuffle4(this.source3, this.destination4); + + /// + /// Executes the ZYXW three-to-four operator. + /// + [Benchmark] + public void Pad4Zyxw() => SimdUtils.Pad3Shuffle4(this.source3, this.destination4); + + /// + /// Executes the ZYX three-to-three operator. + /// + [Benchmark] + public void Shuffle3Zyx() => SimdUtils.Shuffle3(this.source3, this.destination3); +} diff --git a/tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs b/tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs index 12b4c3426..4bf1a0b66 100644 --- a/tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs +++ b/tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs @@ -303,50 +303,30 @@ public partial class SimdUtilsTests { int size = FeatureTestRunner.Deserialize(serialized); - // These cannot be expressed as a theory as you cannot - // use RemoteExecutor within generic methods nor pass - // IShuffle4 to the generic utils method. - WXYZShuffle4 wxyz = default; TestShuffleByte4Channel( size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, wxyz), + (s, d) => SimdUtils.Shuffle4(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle2103); - WZYXShuffle4 wzyx = default; TestShuffleByte4Channel( size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, wzyx), + (s, d) => SimdUtils.Shuffle4(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0123); - YZWXShuffle4 yzwx = default; TestShuffleByte4Channel( size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, yzwx), + (s, d) => SimdUtils.Shuffle4(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0321); - ZYXWShuffle4 zyxw = default; TestShuffleByte4Channel( size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, zyxw), + (s, d) => SimdUtils.Shuffle4(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3012); - DefaultShuffle4 xwyz = new(SimdUtils.Shuffle.MMShuffle2130); TestShuffleByte4Channel( size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, xwyz), - xwyz.Control); - - DefaultShuffle4 yyyy = new(SimdUtils.Shuffle.MMShuffle1111); - TestShuffleByte4Channel( - size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, yyyy), - yyyy.Control); - - DefaultShuffle4 wwww = new(SimdUtils.Shuffle.MMShuffle3333); - TestShuffleByte4Channel( - size, - (s, d) => SimdUtils.Shuffle4(s.Span, d.Span, wwww), - wwww.Control); + (s, d) => SimdUtils.Shuffle4(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle1230); } FeatureTestRunner.RunWithHwIntrinsicsFeature( @@ -363,32 +343,10 @@ public partial class SimdUtilsTests { int size = FeatureTestRunner.Deserialize(serialized); - // These cannot be expressed as a theory as you cannot - // use RemoteExecutor within generic methods nor pass - // IShuffle3 to the generic utils method. - DefaultShuffle3 zyx = new(SimdUtils.Shuffle.MMShuffle3012); - TestShuffleByte3Channel( - size, - (s, d) => SimdUtils.Shuffle3(s.Span, d.Span, zyx), - zyx.Control); - - DefaultShuffle3 xyz = new(SimdUtils.Shuffle.MMShuffle3210); - TestShuffleByte3Channel( - size, - (s, d) => SimdUtils.Shuffle3(s.Span, d.Span, xyz), - xyz.Control); - - DefaultShuffle3 yyy = new(SimdUtils.Shuffle.MMShuffle3111); - TestShuffleByte3Channel( - size, - (s, d) => SimdUtils.Shuffle3(s.Span, d.Span, yyy), - yyy.Control); - - DefaultShuffle3 zzz = new(SimdUtils.Shuffle.MMShuffle3222); TestShuffleByte3Channel( size, - (s, d) => SimdUtils.Shuffle3(s.Span, d.Span, zzz), - zzz.Control); + (s, d) => SimdUtils.Shuffle3(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle3012); } FeatureTestRunner.RunWithHwIntrinsicsFeature( @@ -405,32 +363,25 @@ public partial class SimdUtilsTests { int size = FeatureTestRunner.Deserialize(serialized); - // These cannot be expressed as a theory as you cannot - // use RemoteExecutor within generic methods nor pass - // IPad3Shuffle4 to the generic utils method. - XYZWPad3Shuffle4 xyzw = default; TestPad3Shuffle4Channel( size, - (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, xyzw), + (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3210); - DefaultPad3Shuffle4 xwyz = new(SimdUtils.Shuffle.MMShuffle2130); TestPad3Shuffle4Channel( size, - (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, xwyz), - xwyz.Control); + (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle2103); - DefaultPad3Shuffle4 yyyy = new(SimdUtils.Shuffle.MMShuffle1111); TestPad3Shuffle4Channel( size, - (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, yyyy), - yyyy.Control); + (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle0123); - DefaultPad3Shuffle4 wwww = new(SimdUtils.Shuffle.MMShuffle3333); TestPad3Shuffle4Channel( size, - (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, wwww), - wwww.Control); + (s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle3012); } FeatureTestRunner.RunWithHwIntrinsicsFeature( @@ -447,32 +398,25 @@ public partial class SimdUtilsTests { int size = FeatureTestRunner.Deserialize(serialized); - // These cannot be expressed as a theory as you cannot - // use RemoteExecutor within generic methods nor pass - // IShuffle4Slice3 to the generic utils method. - XYZWShuffle4Slice3 xyzw = default; TestShuffle4Slice3Channel( size, - (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, xyzw), + (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3210); - DefaultShuffle4Slice3 xwyz = new(SimdUtils.Shuffle.MMShuffle2130); TestShuffle4Slice3Channel( size, - (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, xwyz), - xwyz.Control); + (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle0321); - DefaultShuffle4Slice3 yyyy = new(SimdUtils.Shuffle.MMShuffle1111); TestShuffle4Slice3Channel( size, - (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, yyyy), - yyyy.Control); + (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle0123); - DefaultShuffle4Slice3 wwww = new(SimdUtils.Shuffle.MMShuffle3333); TestShuffle4Slice3Channel( size, - (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, wwww), - wwww.Control); + (s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span), + SimdUtils.Shuffle.MMShuffle3012); } FeatureTestRunner.RunWithHwIntrinsicsFeature(