tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Normalize packed pixel conversion shuffles

pull/3161/head
James Jackson-South 3 weeks ago
parent
873c64ec12
commit
5a2fbbe795
14 changed files with 1157 additions and 478 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 32
      src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs
  2. 131
      src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
  3. 56
      src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
  4. 395
      src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs
  5. 130
      src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
  6. 366
      src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs
  7. 60
      src/ImageSharp/PixelFormats/Utils/PixelConverter.cs
  8. 5
      tests/ImageSharp.Benchmarks/Bulk/Pad3Shuffle4Channel.cs
  9. 3
      tests/ImageSharp.Benchmarks/Bulk/Shuffle3Channel.cs
  10. 5
      tests/ImageSharp.Benchmarks/Bulk/Shuffle4Slice3Channel.cs
  11. 2
      tests/ImageSharp.Benchmarks/Bulk/ShuffleByte4Channel.cs
  12. 222
      tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversion.cs
  13. 128
      tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs
  14. 100
      tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs

32
src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs
View File

@ -1,36 +1,26 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
// The JIT can detect and optimize rotation idioms ROTL (Rotate Left)
// and ROTR (Rotate Right) emitting efficient CPU instructions:
// https://github.com/dotnet/coreclr/pull/1830
using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp;
/// <summary>
/// Defines the contract for methods that allow the shuffling of pixel components.
/// Used for shuffling on platforms that do not support Hardware Intrinsics.
/// Defines a stateless operation over packed pixel components.
/// </summary>
internal interface IComponentShuffle
{
/// <summary>
/// Shuffles then slices 8-bit integers in <paramref name="source"/>
/// using a byte control and store the results in <paramref name="destination"/>.
/// If successful, this method will reduce the length of <paramref name="source"/> length
/// by the shuffle amount.
/// Reorders one packed pixel.
/// </summary>
/// <param name="source">The source span of bytes.</param>
/// <param name="destination">The destination span of bytes.</param>
void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination);
/// <param name="source">The source components, with the first component in the least-significant byte.</param>
/// <returns>The reordered packed components.</returns>
static abstract uint Invoke(uint source);
/// <summary>
/// Shuffle 8-bit integers in <paramref name="source"/>
/// using the control and store the results in <paramref name="destination"/>.
/// Reorders the packed pixels in a 128-bit vector.
/// </summary>
/// <param name="source">The source span of bytes.</param>
/// <param name="destination">The destination span of bytes.</param>
/// <remarks>
/// Implementation can assume that source.Length is less or equal than destination.Length.
/// Loops should iterate using source.Length.
/// </remarks>
void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination);
/// <param name="source">The source pixels.</param>
/// <returns>The reordered pixels.</returns>
static abstract Vector128<byte> Invoke(Vector128<byte> source);
}

131
src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
View File

@ -1,99 +1,82 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Diagnostics.CodeAnalysis;
using System.Buffers.Binary;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
/// <inheritdoc/>
/// <summary>
/// Defines a stateless operation that reorders a three-component pixel after adding opaque alpha.
/// </summary>
internal interface IPad3Shuffle4 : IComponentShuffle
{
}
internal readonly struct DefaultPad3Shuffle4([ConstantExpected] byte control) : IPad3Shuffle4
/// <summary>
/// Preserves XYZ order and appends opaque W.
/// </summary>
internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4
{
public byte Control { get; } = control;
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
#pragma warning disable CA1857 // A constant is expected for the parameter
=> HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, this.Control);
#pragma warning restore CA1857 // A constant is expected for the parameter
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(destination);
SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0);
public static uint Invoke(uint source) => source;
for (nuint i = 0, j = 0; i < (uint)source.Length; i += 3, j += 4)
{
// Expanding 3-byte pixels to 4 bytes can overwrite the next source
// triplet when spans overlap. Assemble the padded pixel first, then
// shuffle from the staged uint.
uint packed =
Unsafe.Add(ref sBase, i + 0u) |
((uint)Unsafe.Add(ref sBase, i + 1u) << 8) |
((uint)Unsafe.Add(ref sBase, i + 2u) << 16) |
0xFF000000;
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) => source;
}
ref byte pBase = ref Unsafe.As<uint, byte>(ref packed);
/// <summary>
/// Reorders padded XYZW components to WXYZ.
/// </summary>
internal readonly struct WXYZPad3Shuffle4 : IPad3Shuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) => BitOperations.RotateLeft(source, 8);
Unsafe.Add(ref dBase, j + 0u) = Unsafe.Add(ref pBase, p0);
Unsafe.Add(ref dBase, j + 1u) = Unsafe.Add(ref pBase, p1);
Unsafe.Add(ref dBase, j + 2u) = Unsafe.Add(ref pBase, p2);
Unsafe.Add(ref dBase, j + 3u) = Unsafe.Add(ref pBase, p3);
}
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14));
}
internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4
/// <summary>
/// Reorders padded XYZW components to WZYX.
/// </summary>
internal readonly struct WZYXPad3Shuffle4 : IPad3Shuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210);
public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source);
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
}
/// <summary>
/// Reorders padded XYZW components to ZYXW.
/// </summary>
internal readonly struct ZYXWPad3Shuffle4 : IPad3Shuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
public static uint Invoke(uint source)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(destination);
ref byte sEnd = ref Unsafe.Add(ref sBase, (uint)source.Length);
ref byte sLoopEnd = ref Unsafe.Subtract(ref sEnd, 4);
while (Unsafe.IsAddressLessThan(ref sBase, ref sLoopEnd))
{
// The fast scalar path reads one extra byte past the source triplet.
// Keep that widened read in a local before writing the expanded pixel
// so overlapping destinations cannot change what was read.
uint packed = Unsafe.As<byte, uint>(ref sBase) | 0xFF000000;
Unsafe.As<byte, uint>(ref dBase) = packed;
sBase = ref Unsafe.Add(ref sBase, 3);
dBase = ref Unsafe.Add(ref dBase, 4);
}
while (Unsafe.IsAddressLessThan(ref sBase, ref sEnd))
{
// The final triplet cannot use the widened read above, so assemble
// the same padded uint byte-by-byte before the overlapping store.
uint packed =
Unsafe.Add(ref sBase, 0u) |
((uint)Unsafe.Add(ref sBase, 1u) << 8) |
((uint)Unsafe.Add(ref sBase, 2u) << 16) |
0xFF000000;
Unsafe.As<byte, uint>(ref dBase) = packed;
sBase = ref Unsafe.Add(ref sBase, 3);
dBase = ref Unsafe.Add(ref dBase, 4);
}
// Preserve opaque W and Y while exchanging X and Z.
uint wy = source & 0xFF00FF00;
uint xz = source & 0x00FF00FF;
return wy | BitOperations.RotateLeft(xz, 16);
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
}

56
src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
View File

@ -1,51 +1,37 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
/// <inheritdoc/>
/// <summary>
/// Identifies a stateless three-component shuffle operator.
/// </summary>
internal interface IShuffle3 : IComponentShuffle
{
}
internal readonly struct DefaultShuffle3([ConstantExpected] byte control) : IShuffle3
/// <summary>
/// Reorders XYZ components to ZYX.
/// </summary>
internal readonly struct ZYXShuffle3 : IShuffle3
{
public byte Control { get; } = control;
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
#pragma warning disable CA1857 // A constant is expected for the parameter
=> HwIntrinsics.Shuffle3Reduce(ref source, ref destination, this.Control);
#pragma warning restore CA1857 // A constant is expected for the parameter
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
public static uint Invoke(uint source)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(destination);
SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0);
for (nuint i = 0; i < (uint)source.Length; i += 3)
{
// The scalar remainder can run in-place after the vector body. Load
// the full 3-byte pixel into a register-sized value before stores so
// channel swaps cannot corrupt later reads from the same pixel.
uint packed =
Unsafe.Add(ref sBase, i + 0u) |
((uint)Unsafe.Add(ref sBase, i + 1u) << 8) |
((uint)Unsafe.Add(ref sBase, i + 2u) << 16);
ref byte pBase = ref Unsafe.As<uint, byte>(ref packed);
Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0);
Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1);
Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2);
}
// Y is already centered; shift X and Z directly into each other's byte positions.
uint y = source & 0x0000FF00;
uint x = (source & 0x000000FF) << 16;
uint z = (source & 0x00FF0000) >> 16;
return x | y | z;
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
}

395
src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs
View File

@ -2,183 +2,320 @@
// Licensed under the Six Labors Split License.
using System.Buffers.Binary;
using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
/// <inheritdoc/>
/// <summary>
/// Defines a stateless operation over one packed four-component pixel.
/// </summary>
internal interface IShuffle4 : IComponentShuffle
{
/// <summary>
/// Reorders the packed pixels in a 256-bit vector.
/// </summary>
/// <param name="source">The source pixels.</param>
/// <returns>The reordered pixels.</returns>
static abstract Vector256<byte> Invoke(Vector256<byte> source);
/// <summary>
/// Reorders the packed pixels in a 512-bit vector.
/// </summary>
/// <param name="source">The source pixels.</param>
/// <returns>The reordered pixels.</returns>
static abstract Vector512<byte> Invoke(Vector512<byte> source);
}
internal readonly struct DefaultShuffle4([ConstantExpected] byte control) : IShuffle4
/// <summary>
/// Reorders XYZW components to WXYZ.
/// </summary>
internal readonly struct WXYZShuffle4 : IShuffle4
{
public byte Control { get; } = control;
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
#pragma warning disable CA1857 // A constant is expected for the parameter
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, this.Control);
#pragma warning restore CA1857 // A constant is expected for the parameter
public static uint Invoke(uint source)
{
// source = [W Z Y X]
// ROTL(8, source) = [Z Y X W]
return BitOperations.RotateLeft(source, 8);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask());
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(destination);
SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0);
for (nuint i = 0; i < (uint)source.Length; i += 4)
{
// The generic path may be used with source and destination pointing
// at the same pixel. Load all channels first so subsequent stores
// index only staged bytes, matching the specialized uint shuffles.
uint packed = Unsafe.As<byte, uint>(ref Unsafe.Add(ref sBase, i));
ref byte pBase = ref Unsafe.As<uint, byte>(ref packed);
Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0);
Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1);
Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2);
Unsafe.Add(ref dBase, i + 3u) = Unsafe.Add(ref pBase, p3);
}
// AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
Vector128<byte> mask = CreateMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
}
internal readonly struct WXYZShuffle4 : IShuffle4
{
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle2103);
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
/// <summary>
/// Creates the indices that rotate each XYZW pixel to WXYZ within one 128-bit lane.
/// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask()
=> Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14);
/// <summary>
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
/// </summary>
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
{
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source));
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination));
uint n = (uint)source.Length / 4;
for (nuint i = 0; i < n; i++)
{
uint packed = Unsafe.Add(ref sBase, i);
// packed = [W Z Y X]
// ROTL(8, packed) = [Z Y X W]
Unsafe.Add(ref dBase, i) = (packed << 8) | (packed >> 24);
}
// Native vpshufb ignores the lane offsets, while Vector512.Shuffle treats the indices as
// absolute. Encoding both meanings keeps the AVX-512 and managed fallback results identical.
return Vector512.Create(
0x0605040702010003UL,
0x0E0D0C0F0A09080BUL,
0x1615141712111013UL,
0x1E1D1C1F1A19181BUL,
0x2625242722212023UL,
0x2E2D2C2F2A29282BUL,
0x3635343732313033UL,
0x3E3D3C3F3A39383BUL).AsByte();
}
}
/// <summary>
/// Reorders XYZW components to WZYX.
/// </summary>
internal readonly struct WZYXShuffle4 : IShuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0123);
public static uint Invoke(uint source)
{
// Reversing the integer's endianness also reverses the four byte components.
return BinaryPrimitives.ReverseEndianness(source);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask());
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source)
{
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source));
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination));
uint n = (uint)source.Length / 4;
for (nuint i = 0; i < n; i++)
{
uint packed = Unsafe.Add(ref sBase, i);
// packed = [W Z Y X]
// REVERSE(packedArgb) = [X Y Z W]
Unsafe.Add(ref dBase, i) = BinaryPrimitives.ReverseEndianness(packed);
}
Vector128<byte> mask = CreateMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
/// <summary>
/// Creates the indices that reverse each XYZW pixel to WZYX within one 128-bit lane.
/// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask()
=> Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12);
/// <summary>
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
/// </summary>
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0405060700010203UL,
0x0C0D0E0F08090A0BUL,
0x1415161710111213UL,
0x1C1D1E1F18191A1BUL,
0x2425262720212223UL,
0x2C2D2E2F28292A2BUL,
0x3435363730313233UL,
0x3C3D3E3F38393A3BUL).AsByte();
}
/// <summary>
/// Reorders XYZW components to YZWX.
/// </summary>
internal readonly struct YZWXShuffle4 : IShuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0321);
public static uint Invoke(uint source)
{
// source = [W Z Y X]
// ROTR(8, source) = [X W Z Y]
return BitOperations.RotateRight(source, 8);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask());
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source)
{
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source));
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination));
uint n = (uint)source.Length / 4;
for (nuint i = 0; i < n; i++)
{
uint packed = Unsafe.Add(ref sBase, i);
// packed = [W Z Y X]
// ROTR(8, packedArgb) = [Y Z W X]
Unsafe.Add(ref dBase, i) = BitOperations.RotateRight(packed, 8);
}
Vector128<byte> mask = CreateMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
/// <summary>
/// Creates the indices that rotate each XYZW pixel to YZWX within one 128-bit lane.
/// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask()
=> Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12);
/// <summary>
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
/// </summary>
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0407060500030201UL,
0x0C0F0E0D080B0A09UL,
0x1417161510131211UL,
0x1C1F1E1D181B1A19UL,
0x2427262520232221UL,
0x2C2F2E2D282B2A29UL,
0x3437363530333231UL,
0x3C3F3E3D383B3A39UL).AsByte();
}
/// <summary>
/// Reorders XYZW components to ZYXW.
/// </summary>
internal readonly struct ZYXWShuffle4 : IShuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3012);
public static uint Invoke(uint source)
{
// Preserve W and Y while rotating the masked X/Z bytes into each other's positions.
uint wy = source & 0xFF00FF00;
uint xz = source & 0x00FF00FF;
return wy | BitOperations.RotateLeft(xz, 16);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask());
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source)
{
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source));
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination));
uint n = (uint)source.Length / 4;
for (nuint i = 0; i < n; i++)
{
uint packed = Unsafe.Add(ref sBase, i);
// packed = [W Z Y X]
// tmp1 = [W 0 Y 0]
// tmp2 = [0 Z 0 X]
// tmp3=ROTL(16, tmp2) = [0 X 0 Z]
// tmp1 + tmp3 = [W X Y Z]
uint tmp1 = packed & 0xFF00FF00;
uint tmp2 = packed & 0x00FF00FF;
uint tmp3 = BitOperations.RotateLeft(tmp2, 16);
Unsafe.Add(ref dBase, i) = tmp1 + tmp3;
}
Vector128<byte> mask = CreateMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
/// <summary>
/// Creates the indices that exchange X and Z in each XYZW pixel within one 128-bit lane.
/// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask()
=> Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15);
/// <summary>
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
/// </summary>
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0704050603000102UL,
0x0F0C0D0E0B08090AUL,
0x1714151613101112UL,
0x1F1C1D1E1B18191AUL,
0x2724252623202122UL,
0x2F2C2D2E2B28292AUL,
0x3734353633303132UL,
0x3F3C3D3E3B38393AUL).AsByte();
}
/// <summary>
/// Reorders XYZW components to XWZY.
/// </summary>
internal readonly struct XWZYShuffle4 : IShuffle4
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle1230);
public static uint Invoke(uint source)
{
// Preserve X and Z while rotating the masked Y/W bytes into each other's positions.
uint xz = source & 0x00FF00FF;
uint yw = source & 0xFF00FF00;
return xz | BitOperations.RotateLeft(yw, 16);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask());
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source)
{
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source));
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination));
uint n = (uint)source.Length / 4;
for (nuint i = 0; i < n; i++)
{
uint packed = Unsafe.Add(ref sBase, i);
// packed = [W Z Y X]
// tmp1 = [0 Z 0 X]
// tmp2 = [W 0 Y 0]
// tmp3=ROTL(16, tmp2) = [Y 0 W 0]
// tmp1 + tmp3 = [Y Z W X]
uint tmp1 = packed & 0x00FF00FF;
uint tmp2 = packed & 0xFF00FF00;
uint tmp3 = BitOperations.RotateLeft(tmp2, 16);
Unsafe.Add(ref dBase, i) = tmp1 + tmp3;
}
Vector128<byte> mask = CreateMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
/// <summary>
/// Creates the indices that exchange Y and W in each XYZW pixel within one 128-bit lane.
/// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask()
=> Vector128.Create((byte)0, 3, 2, 1, 4, 7, 6, 5, 8, 11, 10, 9, 12, 15, 14, 13);
/// <summary>
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector.
/// </summary>
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0506070401020300UL,
0x0D0E0F0C090A0B08UL,
0x1516171411121310UL,
0x1D1E1F1C191A1B18UL,
0x2526272421222320UL,
0x2D2E2F2C292A2B28UL,
0x3536373431323330UL,
0x3D3E3F3C393A3B38UL).AsByte();
}

130
src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
View File

@ -1,101 +1,85 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Diagnostics.CodeAnalysis;
using System.Buffers.Binary;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
/// <inheritdoc/>
/// <summary>
/// Defines a stateless operation that reorders four packed components before retaining three.
/// </summary>
internal interface IShuffle4Slice3 : IComponentShuffle
{
}
internal readonly struct DefaultShuffle4Slice3([ConstantExpected] byte control) : IShuffle4Slice3
/// <summary>
/// Preserves XYZ order and discards W.
/// </summary>
internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3
{
public byte Control { get; } = control;
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
#pragma warning disable CA1857 // A constant is expected for the parameter
=> HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, this.Control);
#pragma warning restore CA1857 // A constant is expected for the parameter
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(destination);
public static uint Invoke(uint source) => source;
SimdUtils.Shuffle.InverseMMShuffle(this.Control, out _, out uint p2, out uint p1, out uint p0);
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) => source;
}
for (nuint i = 0, j = 0; i < (uint)destination.Length; i += 3, j += 4)
{
// Shrinking 4-byte pixels to 3 bytes can still be called in-place by
// tail code. Read the complete source pixel first, then write only
// the requested channels into the destination triplet.
uint packed = Unsafe.As<byte, uint>(ref Unsafe.Add(ref sBase, j));
ref byte pBase = ref Unsafe.As<uint, byte>(ref packed);
/// <summary>
/// Reorders XYZW components to YZW before discarding X.
/// </summary>
internal readonly struct YZWXShuffle4Slice3 : IShuffle4Slice3
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) => BitOperations.RotateRight(source, 8);
Unsafe.Add(ref dBase, i + 0u) = Unsafe.Add(ref pBase, p0);
Unsafe.Add(ref dBase, i + 1u) = Unsafe.Add(ref pBase, p1);
Unsafe.Add(ref dBase, i + 2u) = Unsafe.Add(ref pBase, p2);
}
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12));
}
internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3
/// <summary>
/// Reorders XYZW components to WZY before discarding X.
/// </summary>
internal readonly struct WZYXShuffle4Slice3 : IShuffle4Slice3
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination)
=> HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3210);
public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source);
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
}
/// <summary>
/// Reorders XYZW components to ZYX before discarding W.
/// </summary>
internal readonly struct ZYXWShuffle4Slice3 : IShuffle4Slice3
{
/// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)]
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination)
public static uint Invoke(uint source)
{
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source));
ref Byte3 dBase = ref Unsafe.As<byte, Byte3>(ref MemoryMarshal.GetReference(destination));
nint n = (nint)(uint)source.Length / 4;
nint m = Numerics.Modulo4(n);
nint u = n - m;
ref uint sLoopEnd = ref Unsafe.Add(ref sBase, u);
ref uint sEnd = ref Unsafe.Add(ref sBase, n);
while (Unsafe.IsAddressLessThan(ref sBase, ref sLoopEnd))
{
// Stage the four source pixels before the 3-byte stores. Even
// though this path preserves XYZ order, the packed loads must happen
// before destination writes when the spans overlap.
uint packed0 = Unsafe.Add(ref sBase, 0u);
uint packed1 = Unsafe.Add(ref sBase, 1u);
uint packed2 = Unsafe.Add(ref sBase, 2u);
uint packed3 = Unsafe.Add(ref sBase, 3u);
Unsafe.Add(ref dBase, 0u) = Unsafe.As<uint, Byte3>(ref packed0);
Unsafe.Add(ref dBase, 1u) = Unsafe.As<uint, Byte3>(ref packed1);
Unsafe.Add(ref dBase, 2u) = Unsafe.As<uint, Byte3>(ref packed2);
Unsafe.Add(ref dBase, 3u) = Unsafe.As<uint, Byte3>(ref packed3);
sBase = ref Unsafe.Add(ref sBase, 4);
dBase = ref Unsafe.Add(ref dBase, 4);
}
while (Unsafe.IsAddressLessThan(ref sBase, ref sEnd))
{
// Same overlap rule as the unrolled loop: take the 4-byte source
// pixel before storing the 3-byte destination value.
uint packed = Unsafe.Add(ref sBase, 0u);
Unsafe.Add(ref dBase, 0u) = Unsafe.As<uint, Byte3>(ref packed);
sBase = ref Unsafe.Add(ref sBase, 1);
dBase = ref Unsafe.Add(ref dBase, 1);
}
// Preserve W and Y while exchanging X and Z; W is subsequently discarded.
uint wy = source & 0xFF00FF00;
uint xz = source & 0x00FF00FF;
return wy | BitOperations.RotateLeft(xz, 16);
}
/// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
}
[StructLayout(LayoutKind.Explicit, Size = 3)]

366
src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs
View File

@ -6,6 +6,8 @@ using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp;
@ -42,22 +44,104 @@ internal static partial class SimdUtils
/// <typeparam name="TShuffle">The type of shuffle struct.</typeparam>
/// <param name="source">The source span of bytes.</param>
/// <param name="destination">The destination span of bytes.</param>
/// <param name="shuffle">The type of shuffle to perform.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void Shuffle4<TShuffle>(
ReadOnlySpan<byte> source,
Span<byte> destination,
TShuffle shuffle)
Span<byte> destination)
where TShuffle : struct, IShuffle4
{
VerifyShuffle4SpanInput(source, destination);
shuffle.ShuffleReduce(ref source, ref destination);
ref byte sourceBase = ref MemoryMarshal.GetReference(source);
ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
int length = source.Length;
int i = 0;
// Deal with the remainder:
if (source.Length > 0)
// The same offset flows through descending widths. This keeps a single traversal while
// allowing a row that is not a multiple of the widest register to retain a vectorized tail.
if (Vector512.IsHardwareAccelerated)
{
int fourVectorsFromEnd = length - (Vector512<byte>.Count * 4);
for (; i <= fourVectorsFromEnd; i += Vector512<byte>.Count * 4)
{
// Four independent vectors amortize loop control and expose enough work for the CPU
// to overlap loads, byte shuffles, and stores without changing pixel ordering.
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i))
.StoreUnsafe(ref destinationBase, (nuint)i);
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + Vector512<byte>.Count)))
.StoreUnsafe(ref destinationBase, (nuint)(i + Vector512<byte>.Count));
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512<byte>.Count * 2))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512<byte>.Count * 2)));
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512<byte>.Count * 3))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512<byte>.Count * 3)));
}
int oneVectorFromEnd = length - Vector512<byte>.Count;
for (; i <= oneVectorFromEnd; i += Vector512<byte>.Count)
{
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i))
.StoreUnsafe(ref destinationBase, (nuint)i);
}
}
if (Vector256.IsHardwareAccelerated)
{
int fourVectorsFromEnd = length - (Vector256<byte>.Count * 4);
for (; i <= fourVectorsFromEnd; i += Vector256<byte>.Count * 4)
{
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i))
.StoreUnsafe(ref destinationBase, (nuint)i);
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + Vector256<byte>.Count)))
.StoreUnsafe(ref destinationBase, (nuint)(i + Vector256<byte>.Count));
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256<byte>.Count * 2))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256<byte>.Count * 2)));
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256<byte>.Count * 3))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256<byte>.Count * 3)));
}
int oneVectorFromEnd = length - Vector256<byte>.Count;
for (; i <= oneVectorFromEnd; i += Vector256<byte>.Count)
{
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i))
.StoreUnsafe(ref destinationBase, (nuint)i);
}
}
if (Vector128.IsHardwareAccelerated)
{
shuffle.Shuffle(source, destination);
int fourVectorsFromEnd = length - (Vector128<byte>.Count * 4);
for (; i <= fourVectorsFromEnd; i += Vector128<byte>.Count * 4)
{
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i))
.StoreUnsafe(ref destinationBase, (nuint)i);
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + Vector128<byte>.Count)))
.StoreUnsafe(ref destinationBase, (nuint)(i + Vector128<byte>.Count));
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128<byte>.Count * 2))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128<byte>.Count * 2)));
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128<byte>.Count * 3))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128<byte>.Count * 3)));
}
int oneVectorFromEnd = length - Vector128<byte>.Count;
for (; i <= oneVectorFromEnd; i += Vector128<byte>.Count)
{
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i))
.StoreUnsafe(ref destinationBase, (nuint)i);
}
}
// The vector cascade leaves fewer than four pixels. A full uint load keeps each pixel
// in a register while the closed operator resolves to its rotate, reverse, or mask sequence.
for (; i < length; i += 4)
{
uint packed = Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)i));
Unsafe.As<byte, uint>(ref Unsafe.Add(ref destinationBase, (nuint)i)) = TShuffle.Invoke(packed);
}
}
@ -68,23 +152,112 @@ internal static partial class SimdUtils
/// <typeparam name="TShuffle">The type of shuffle struct.</typeparam>
/// <param name="source">The source span of bytes.</param>
/// <param name="destination">The destination span of bytes.</param>
/// <param name="shuffle">The type of shuffle to perform.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void Shuffle3<TShuffle>(
ReadOnlySpan<byte> source,
Span<byte> destination,
TShuffle shuffle)
Span<byte> destination)
where TShuffle : struct, IShuffle3
{
// Source length should be smaller than destination length, and divisible by 3.
VerifyShuffle3SpanInput(source, destination);
shuffle.ShuffleReduce(ref source, ref destination);
ref byte sourceBase = ref MemoryMarshal.GetReference(source);
ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
int length = source.Length;
int i = 0;
// Deal with the remainder:
if (source.Length > 0)
if (Vector128.IsHardwareAccelerated)
{
// Each group contains sixteen XYZ pixels in three registers. The pad mask expands
// four triplets per register to XYZW, with 0x80 selecting zero for the temporary W lane.
Vector128<byte> padMask = Vector128.Create(
(byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
// After the operator has reordered padded pixels, these masks remove every temporary
// W lane and repack the four registers into three contiguous XYZ destination registers.
Vector128<byte> sliceMask = Vector128.Create(
(byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
Vector128<byte> sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12);
ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase);
ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase);
nuint sourceVectorCount = (uint)length / (uint)Vector128<byte>.Count;
nuint vectorIndex = 0;
for (; vectorIndex + 2 < sourceVectorCount; vectorIndex += 3)
{
// Realign the three source registers into four registers holding four complete
// triplets apiece. All source registers are captured before any destination store.
ref Vector128<byte> source0 = ref Unsafe.Add(ref sourceVectors, vectorIndex);
Vector128<byte> v0 = source0;
Vector128<byte> v1 = Unsafe.Add(ref source0, 1);
Vector128<byte> v2 = Unsafe.Add(ref source0, 2);
Vector128<byte> v3 = Vector128_.ShiftRightBytesInVector(v2, 4);
v2 = Vector128_.AlignRight(v2, v1, 8);
v1 = Vector128_.AlignRight(v1, v0, 12);
v0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask));
v1 = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask));
v2 = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask));
v3 = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask));
v0 = Vector128_.ShuffleNative(v0, sliceEndMask);
v1 = Vector128_.ShuffleNative(v1, sliceMask);
v2 = Vector128_.ShuffleNative(v2, sliceEndMask);
v3 = Vector128_.ShuffleNative(v3, sliceMask);
Vector128<byte> destination0 = Vector128_.AlignRight(v1, v0, 4);
Vector128<byte> destination2 = Vector128_.AlignRight(v3, v2, 12);
v1 = Vector128_.ShiftLeftBytesInVector(v1, 4);
v2 = Vector128_.ShiftRightBytesInVector(v2, 4);
Vector128<byte> destination1 = Vector128_.AlignRight(v2, v1, 8);
ref Vector128<byte> destination0Ref = ref Unsafe.Add(ref destinationVectors, vectorIndex);
destination0Ref = destination0;
Unsafe.Add(ref destination0Ref, 1) = destination1;
Unsafe.Add(ref destination0Ref, 2) = destination2;
}
i = (int)(vectorIndex * (uint)Vector128<byte>.Count);
int oneTailVectorFromEnd = length - Vector128<byte>.Count;
for (; i <= oneTailVectorFromEnd; i += 12)
{
// A single readable register contains four complete triplets plus four bytes from
// the following pixels. The pad mask ignores those extra bytes before the operator
// runs, and the slice mask packs the four results into the low twelve bytes.
Vector128<byte> result = Vector128.LoadUnsafe(ref sourceBase, (nuint)i);
result = Vector128_.ShuffleNative(result, padMask);
result = TShuffle.Invoke(result);
result = Vector128_.ShuffleNative(result, sliceMask);
// Store exactly twelve bytes so an in-place shuffle does not overwrite the next
// source triplet captured by the following iteration.
Unsafe.As<byte, Vector64<byte>>(ref Unsafe.Add(ref destinationBase, (nuint)i)) = result.GetLower();
Unsafe.As<byte, uint>(ref Unsafe.Add(ref destinationBase, (nuint)(i + 8))) = result.AsUInt32().GetElement(2);
}
}
int widenedReadEnd = length - 3;
for (; i < widenedReadEnd; i += 3)
{
shuffle.Shuffle(source, destination);
// The fourth byte belongs to the following pixel, but the operator only contributes the
// low three result bytes. This unaligned read replaces three dependent byte loads safely.
uint packed = Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)i));
uint shuffled = TShuffle.Invoke(packed);
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)i)) = Unsafe.As<uint, Byte3>(ref shuffled);
}
if (i < length)
{
// The final triplet has no fourth readable byte, so construct only this terminal pixel.
uint packed =
Unsafe.Add(ref sourceBase, (nuint)i) |
((uint)Unsafe.Add(ref sourceBase, (nuint)(i + 1)) << 8) |
((uint)Unsafe.Add(ref sourceBase, (nuint)(i + 2)) << 16);
uint shuffled = TShuffle.Invoke(packed);
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)i)) = Unsafe.As<uint, Byte3>(ref shuffled);
}
}
@ -95,22 +268,78 @@ internal static partial class SimdUtils
/// <typeparam name="TShuffle">The type of shuffle struct.</typeparam>
/// <param name="source">The source span of bytes.</param>
/// <param name="destination">The destination span of bytes.</param>
/// <param name="shuffle">The type of shuffle to perform.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void Pad3Shuffle4<TShuffle>(
ReadOnlySpan<byte> source,
Span<byte> destination,
TShuffle shuffle)
Span<byte> destination)
where TShuffle : struct, IPad3Shuffle4
{
VerifyPad3Shuffle4SpanInput(source, destination);
shuffle.ShuffleReduce(ref source, ref destination);
ref byte sourceBase = ref MemoryMarshal.GetReference(source);
ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
int sourceLength = source.Length;
int sourceOffset = 0;
int destinationOffset = 0;
// Deal with the remainder:
if (source.Length > 0)
if (Vector128.IsHardwareAccelerated)
{
// The fixed mask expands four XYZ triplets to four XYZW pixels. The zeroed W bytes
// are then filled with opaque alpha before the selected operator reorders each pixel.
Vector128<byte> padMask = Vector128.Create(
(byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
Vector128<byte> opaqueAlpha = Vector128.Create(0xFF000000FF000000UL).AsByte();
ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase);
ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase);
nuint sourceVectorCount = (uint)sourceLength / (uint)Vector128<byte>.Count;
nuint sourceVectorIndex = 0;
nuint destinationVectorIndex = 0;
for (; sourceVectorIndex + 2 < sourceVectorCount;
sourceVectorIndex += 3, destinationVectorIndex += 4)
{
// Three source registers contain sixteen packed triplets. Aligning at 12, 8, and
// 4-byte boundaries produces four registers whose low twelve bytes each hold four pixels.
ref Vector128<byte> source0 = ref Unsafe.Add(ref sourceVectors, sourceVectorIndex);
Vector128<byte> v0 = source0;
Vector128<byte> v1 = Unsafe.Add(ref source0, 1);
Vector128<byte> v2 = Unsafe.Add(ref source0, 2);
Vector128<byte> v3 = Vector128_.ShiftRightBytesInVector(v2, 4);
v2 = Vector128_.AlignRight(v2, v1, 8);
v1 = Vector128_.AlignRight(v1, v0, 12);
ref Vector128<byte> destination0 = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex);
destination0 = TShuffle.Invoke(Vector128_.ShuffleNative(v0, padMask) | opaqueAlpha);
Unsafe.Add(ref destination0, 1) = TShuffle.Invoke(Vector128_.ShuffleNative(v1, padMask) | opaqueAlpha);
Unsafe.Add(ref destination0, 2) = TShuffle.Invoke(Vector128_.ShuffleNative(v2, padMask) | opaqueAlpha);
Unsafe.Add(ref destination0, 3) = TShuffle.Invoke(Vector128_.ShuffleNative(v3, padMask) | opaqueAlpha);
}
sourceOffset = (int)(sourceVectorIndex * (uint)Vector128<byte>.Count);
destinationOffset = (int)(destinationVectorIndex * (uint)Vector128<byte>.Count);
}
int widenedReadEnd = sourceLength - 3;
for (; sourceOffset < widenedReadEnd; sourceOffset += 3, destinationOffset += 4)
{
// The widened load intentionally includes the next pixel's first byte. Replacing that
// high byte with opaque alpha yields the complete XYZW value with one unaligned read.
uint packed = Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset)) | 0xFF000000;
Unsafe.As<byte, uint>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = TShuffle.Invoke(packed);
}
if (sourceOffset < sourceLength)
{
shuffle.Shuffle(source, destination);
// The final triplet cannot use the widened load because no following byte is in range.
uint packed =
Unsafe.Add(ref sourceBase, (nuint)sourceOffset) |
((uint)Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 1)) << 8) |
((uint)Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 2)) << 16) |
0xFF000000;
Unsafe.As<byte, uint>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = TShuffle.Invoke(packed);
}
}
@ -121,22 +350,101 @@ internal static partial class SimdUtils
/// <typeparam name="TShuffle">The type of shuffle struct.</typeparam>
/// <param name="source">The source span of bytes.</param>
/// <param name="destination">The destination span of bytes.</param>
/// <param name="shuffle">The type of shuffle to perform.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void Shuffle4Slice3<TShuffle>(
ReadOnlySpan<byte> source,
Span<byte> destination,
TShuffle shuffle)
Span<byte> destination)
where TShuffle : struct, IShuffle4Slice3
{
VerifyShuffle4Slice3SpanInput(source, destination);
shuffle.ShuffleReduce(ref source, ref destination);
ref byte sourceBase = ref MemoryMarshal.GetReference(source);
ref byte destinationBase = ref MemoryMarshal.GetReference(destination);
int sourceLength = source.Length;
int sourceOffset = 0;
int destinationOffset = 0;
// Deal with the remainder:
if (source.Length > 0)
if (Vector128.IsHardwareAccelerated)
{
// Each operator first places the three retained components in the low bytes of every
// four-byte pixel. These masks then delete the fourth byte and compact sixteen pixels.
Vector128<byte> sliceMask = Vector128.Create(
(byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
Vector128<byte> sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12);
ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase);
ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase);
nuint sourceVectorCount = (uint)sourceLength / (uint)Vector128<byte>.Count;
nuint sourceVectorIndex = 0;
nuint destinationVectorIndex = 0;
for (; sourceVectorIndex + 3 < sourceVectorCount;
sourceVectorIndex += 4, destinationVectorIndex += 3)
{
// Load and transform all sixteen source pixels before writing the shorter output group.
// This preserves forward progress when source and destination begin at the same address.
ref Vector128<byte> source0 = ref Unsafe.Add(ref sourceVectors, sourceVectorIndex);
Vector128<byte> v0 = TShuffle.Invoke(source0);
Vector128<byte> v1 = TShuffle.Invoke(Unsafe.Add(ref source0, 1));
Vector128<byte> v2 = TShuffle.Invoke(Unsafe.Add(ref source0, 2));
Vector128<byte> v3 = TShuffle.Invoke(Unsafe.Add(ref source0, 3));
v0 = Vector128_.ShuffleNative(v0, sliceEndMask);
v1 = Vector128_.ShuffleNative(v1, sliceMask);
v2 = Vector128_.ShuffleNative(v2, sliceEndMask);
v3 = Vector128_.ShuffleNative(v3, sliceMask);
Vector128<byte> destination0 = Vector128_.AlignRight(v1, v0, 4);
Vector128<byte> destination2 = Vector128_.AlignRight(v3, v2, 12);
v1 = Vector128_.ShiftLeftBytesInVector(v1, 4);
v2 = Vector128_.ShiftRightBytesInVector(v2, 4);
Vector128<byte> destination1 = Vector128_.AlignRight(v2, v1, 8);
ref Vector128<byte> destination0Ref = ref Unsafe.Add(ref destinationVectors, destinationVectorIndex);
destination0Ref = destination0;
Unsafe.Add(ref destination0Ref, 1) = destination1;
Unsafe.Add(ref destination0Ref, 2) = destination2;
}
sourceOffset = (int)(sourceVectorIndex * (uint)Vector128<byte>.Count);
destinationOffset = (int)(destinationVectorIndex * (uint)Vector128<byte>.Count);
int oneTailVectorFromEnd = sourceLength - Vector128<byte>.Count;
for (; sourceOffset <= oneTailVectorFromEnd; sourceOffset += 16, destinationOffset += 12)
{
// The operator arranges the three retained components at the front of each pixel.
// One fixed shuffle then compacts four pixels into the low twelve vector bytes.
Vector128<byte> result = TShuffle.Invoke(
Vector128.LoadUnsafe(ref sourceBase, (nuint)sourceOffset));
result = Vector128_.ShuffleNative(result, sliceMask);
// The split store writes the exact 12-byte result and remains safe for in-place shrinking.
Unsafe.As<byte, Vector64<byte>>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = result.GetLower();
Unsafe.As<byte, uint>(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 8))) = result.AsUInt32().GetElement(2);
}
}
int fourPixelsFromEnd = sourceLength - 16;
for (; sourceOffset <= fourPixelsFromEnd; sourceOffset += 16, destinationOffset += 12)
{
// Transform four complete pixels before the first three-byte store. Keeping the source
// values in registers avoids reloads after an in-place shrinking destination advances.
uint packed0 = TShuffle.Invoke(Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset)));
uint packed1 = TShuffle.Invoke(Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 4))));
uint packed2 = TShuffle.Invoke(Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 8))));
uint packed3 = TShuffle.Invoke(Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)(sourceOffset + 12))));
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = Unsafe.As<uint, Byte3>(ref packed0);
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 3))) = Unsafe.As<uint, Byte3>(ref packed1);
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 6))) = Unsafe.As<uint, Byte3>(ref packed2);
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)(destinationOffset + 9))) = Unsafe.As<uint, Byte3>(ref packed3);
}
for (; sourceOffset < sourceLength; sourceOffset += 4, destinationOffset += 3)
{
shuffle.Shuffle(source, destination);
uint packed = TShuffle.Invoke(Unsafe.As<byte, uint>(ref Unsafe.Add(ref sourceBase, (nuint)sourceOffset)));
Unsafe.As<byte, Byte3>(ref Unsafe.Add(ref destinationBase, (nuint)destinationOffset)) = Unsafe.As<uint, Byte3>(ref packed);
}
}

60
src/ImageSharp/PixelFormats/Utils/PixelConverter.cs
View File

@ -33,7 +33,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToArgb32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<WXYZShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<WXYZShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -44,7 +44,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgra32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<ZYXWShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<ZYXWShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -55,7 +55,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToAbgr32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -66,7 +66,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgb24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(source, dest, default);
=> SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -77,7 +77,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgr24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle3012));
=> SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(source, dest);
}
/// <summary>
@ -96,7 +96,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgba32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<YZWXShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<YZWXShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -107,7 +107,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgra32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -118,7 +118,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToAbgr32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<XWZYShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<XWZYShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -129,7 +129,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgb24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0321));
=> SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -140,7 +140,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgr24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0123));
=> SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(source, dest);
}
/// <summary>
@ -159,7 +159,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToArgb32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -170,7 +170,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgba32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<ZYXWShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<ZYXWShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -181,7 +181,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToAbgr32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<WXYZShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<WXYZShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -192,7 +192,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgb24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle3012));
=> SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -203,7 +203,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgr24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(source, dest, default);
=> SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(source, dest);
}
/// <summary>
@ -222,7 +222,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToArgb32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<XWZYShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<XWZYShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -233,7 +233,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgba32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<WZYXShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -244,7 +244,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgra32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4<YZWXShuffle4>(source, dest, default);
=> SimdUtils.Shuffle4<YZWXShuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -255,7 +255,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgb24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0123));
=> SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -266,7 +266,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgr24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle4Slice3(source, dest, new DefaultShuffle4Slice3(SimdUtils.Shuffle.MMShuffle0321));
=> SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(source, dest);
}
/// <summary>
@ -285,7 +285,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgba32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(source, dest, default);
=> SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -296,7 +296,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToArgb32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle2103));
=> SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -307,7 +307,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgra32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle3012));
=> SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -318,7 +318,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToAbgr32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle0123));
=> SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -329,7 +329,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgr24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle3(source, dest, new DefaultShuffle3(SimdUtils.Shuffle.MMShuffle3012));
=> SimdUtils.Shuffle3<ZYXShuffle3>(source, dest);
}
/// <summary>
@ -348,7 +348,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToArgb32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle0123));
=> SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -359,7 +359,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgba32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle3012));
=> SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -370,7 +370,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToBgra32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(source, dest, default);
=> SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -381,7 +381,7 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToAbgr32(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Pad3Shuffle4(source, dest, new DefaultPad3Shuffle4(SimdUtils.Shuffle.MMShuffle2103));
=> SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(source, dest);
/// <summary>
/// Converts a <see cref="ReadOnlySpan{Byte}"/> representing a collection of
@ -392,6 +392,6 @@ internal static class PixelConverter
/// <param name="dest">The destination span of bytes.</param>
[MethodImpl(InliningOptions.ShortMethod)]
public static void ToRgb24(ReadOnlySpan<byte> source, Span<byte> dest)
=> SimdUtils.Shuffle3(source, dest, new DefaultShuffle3(SimdUtils.Shuffle.MMShuffle3012));
=> SimdUtils.Shuffle3<ZYXShuffle3>(source, dest);
}
}

5
tests/ImageSharp.Benchmarks/Bulk/Pad3Shuffle4Channel.cs
View File

@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Benchmarks.Bulk;
[Config(typeof(Config.HwIntrinsics_SSE_AVX))]
public class Pad3Shuffle4Channel
{
private static readonly DefaultPad3Shuffle4 Control = new(SimdUtils.Shuffle.MMShuffle1032);
private byte[] source;
private byte[] destination;
@ -25,11 +24,11 @@ public class Pad3Shuffle4Channel
[Benchmark]
public void Pad3Shuffle4()
=> SimdUtils.Pad3Shuffle4(this.source, this.destination, Control);
=> SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(this.source, this.destination);
[Benchmark]
public void Pad3Shuffle4FastFallback()
=> SimdUtils.Pad3Shuffle4(this.source, this.destination, default(XYZWPad3Shuffle4));
=> SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(this.source, this.destination);
}
// 2020-10-30

3
tests/ImageSharp.Benchmarks/Bulk/Shuffle3Channel.cs
View File

@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Benchmarks.Bulk;
[Config(typeof(Config.HwIntrinsics_SSE_AVX))]
public class Shuffle3Channel
{
private static readonly DefaultShuffle3 Control = new(SimdUtils.Shuffle.MMShuffle3102);
private byte[] source;
private byte[] destination;
@ -25,7 +24,7 @@ public class Shuffle3Channel
[Benchmark]
public void Shuffle3()
=> SimdUtils.Shuffle3(this.source, this.destination, Control);
=> SimdUtils.Shuffle3<ZYXShuffle3>(this.source, this.destination);
}
// 2020-11-02

5
tests/ImageSharp.Benchmarks/Bulk/Shuffle4Slice3Channel.cs
View File

@ -8,7 +8,6 @@ namespace SixLabors.ImageSharp.Benchmarks.Bulk;
[Config(typeof(Config.HwIntrinsics_SSE_AVX))]
public class Shuffle4Slice3Channel
{
private static readonly DefaultShuffle4Slice3 Control = new(SimdUtils.Shuffle.MMShuffle1032);
private byte[] source;
private byte[] destination;
@ -25,11 +24,11 @@ public class Shuffle4Slice3Channel
[Benchmark]
public void Shuffle4Slice3()
=> SimdUtils.Shuffle4Slice3(this.source, this.destination, Control);
=> SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(this.source, this.destination);
[Benchmark]
public void Shuffle4Slice3FastFallback()
=> SimdUtils.Shuffle4Slice3(this.source, this.destination, default(XYZWShuffle4Slice3));
=> SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(this.source, this.destination);
}
// 2020-10-29

2
tests/ImageSharp.Benchmarks/Bulk/ShuffleByte4Channel.cs
View File

@ -24,7 +24,7 @@ public class ShuffleByte4Channel
[Benchmark]
public void Shuffle4Channel()
=> SimdUtils.Shuffle4<WXYZShuffle4>(this.source, this.destination, default);
=> SimdUtils.Shuffle4<WXYZShuffle4>(this.source, this.destination);
}
// 2020-10-29

222
tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversion.cs
View File

@ -0,0 +1,222 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.PixelFormats.Utils;
namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion;
/// <summary>
/// Measures every optimized conversion between the six byte-packed RGB and RGBA pixel layouts.
/// </summary>
[Config(typeof(Config.Short))]
public class PackedPixelConversion
{
private byte[] source3;
private byte[] source4;
private byte[] destination3;
private byte[] destination4;
/// <summary>
/// Gets or sets the number of pixels converted by each invocation.
/// </summary>
[Params(7, 256, 4096)]
public int Count { get; set; }
/// <summary>
/// Creates deterministic source buffers and correctly sized destination buffers.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.source3 = new byte[this.Count * 3];
this.source4 = new byte[this.Count * 4];
this.destination3 = new byte[this.Count * 3];
this.destination4 = new byte[this.Count * 4];
// Non-repeating channel values prevent the JIT or hardware from benefiting from
// zero-filled inputs while keeping both benchmark revisions byte-for-byte identical.
new Random(42).NextBytes(this.source3);
new Random(42).NextBytes(this.source4);
}
/// <summary>
/// Converts RGBA pixels to ARGB pixels.
/// </summary>
[Benchmark]
public void Rgba32ToArgb32() => PixelConverter.FromRgba32.ToArgb32(this.source4, this.destination4);
/// <summary>
/// Converts RGBA pixels to ABGR pixels.
/// </summary>
[Benchmark]
public void Rgba32ToAbgr32() => PixelConverter.FromRgba32.ToAbgr32(this.source4, this.destination4);
/// <summary>
/// Converts RGBA pixels to BGRA pixels.
/// </summary>
[Benchmark]
public void Rgba32ToBgra32() => PixelConverter.FromRgba32.ToBgra32(this.source4, this.destination4);
/// <summary>
/// Converts RGBA pixels to RGB pixels.
/// </summary>
[Benchmark]
public void Rgba32ToRgb24() => PixelConverter.FromRgba32.ToRgb24(this.source4, this.destination3);
/// <summary>
/// Converts RGBA pixels to BGR pixels.
/// </summary>
[Benchmark]
public void Rgba32ToBgr24() => PixelConverter.FromRgba32.ToBgr24(this.source4, this.destination3);
/// <summary>
/// Converts ARGB pixels to RGBA pixels.
/// </summary>
[Benchmark]
public void Argb32ToRgba32() => PixelConverter.FromArgb32.ToRgba32(this.source4, this.destination4);
/// <summary>
/// Converts ARGB pixels to ABGR pixels.
/// </summary>
[Benchmark]
public void Argb32ToAbgr32() => PixelConverter.FromArgb32.ToAbgr32(this.source4, this.destination4);
/// <summary>
/// Converts ARGB pixels to BGRA pixels.
/// </summary>
[Benchmark]
public void Argb32ToBgra32() => PixelConverter.FromArgb32.ToBgra32(this.source4, this.destination4);
/// <summary>
/// Converts ARGB pixels to RGB pixels.
/// </summary>
[Benchmark]
public void Argb32ToRgb24() => PixelConverter.FromArgb32.ToRgb24(this.source4, this.destination3);
/// <summary>
/// Converts ARGB pixels to BGR pixels.
/// </summary>
[Benchmark]
public void Argb32ToBgr24() => PixelConverter.FromArgb32.ToBgr24(this.source4, this.destination3);
/// <summary>
/// Converts ABGR pixels to RGBA pixels.
/// </summary>
[Benchmark]
public void Abgr32ToRgba32() => PixelConverter.FromAbgr32.ToRgba32(this.source4, this.destination4);
/// <summary>
/// Converts ABGR pixels to ARGB pixels.
/// </summary>
[Benchmark]
public void Abgr32ToArgb32() => PixelConverter.FromAbgr32.ToArgb32(this.source4, this.destination4);
/// <summary>
/// Converts ABGR pixels to BGRA pixels.
/// </summary>
[Benchmark]
public void Abgr32ToBgra32() => PixelConverter.FromAbgr32.ToBgra32(this.source4, this.destination4);
/// <summary>
/// Converts ABGR pixels to RGB pixels.
/// </summary>
[Benchmark]
public void Abgr32ToRgb24() => PixelConverter.FromAbgr32.ToRgb24(this.source4, this.destination3);
/// <summary>
/// Converts ABGR pixels to BGR pixels.
/// </summary>
[Benchmark]
public void Abgr32ToBgr24() => PixelConverter.FromAbgr32.ToBgr24(this.source4, this.destination3);
/// <summary>
/// Converts BGRA pixels to RGBA pixels.
/// </summary>
[Benchmark]
public void Bgra32ToRgba32() => PixelConverter.FromBgra32.ToRgba32(this.source4, this.destination4);
/// <summary>
/// Converts BGRA pixels to ARGB pixels.
/// </summary>
[Benchmark]
public void Bgra32ToArgb32() => PixelConverter.FromBgra32.ToArgb32(this.source4, this.destination4);
/// <summary>
/// Converts BGRA pixels to ABGR pixels.
/// </summary>
[Benchmark]
public void Bgra32ToAbgr32() => PixelConverter.FromBgra32.ToAbgr32(this.source4, this.destination4);
/// <summary>
/// Converts BGRA pixels to RGB pixels.
/// </summary>
[Benchmark]
public void Bgra32ToRgb24() => PixelConverter.FromBgra32.ToRgb24(this.source4, this.destination3);
/// <summary>
/// Converts BGRA pixels to BGR pixels.
/// </summary>
[Benchmark]
public void Bgra32ToBgr24() => PixelConverter.FromBgra32.ToBgr24(this.source4, this.destination3);
/// <summary>
/// Converts RGB pixels to RGBA pixels.
/// </summary>
[Benchmark]
public void Rgb24ToRgba32() => PixelConverter.FromRgb24.ToRgba32(this.source3, this.destination4);
/// <summary>
/// Converts RGB pixels to ARGB pixels.
/// </summary>
[Benchmark]
public void Rgb24ToArgb32() => PixelConverter.FromRgb24.ToArgb32(this.source3, this.destination4);
/// <summary>
/// Converts RGB pixels to ABGR pixels.
/// </summary>
[Benchmark]
public void Rgb24ToAbgr32() => PixelConverter.FromRgb24.ToAbgr32(this.source3, this.destination4);
/// <summary>
/// Converts RGB pixels to BGRA pixels.
/// </summary>
[Benchmark]
public void Rgb24ToBgra32() => PixelConverter.FromRgb24.ToBgra32(this.source3, this.destination4);
/// <summary>
/// Converts RGB pixels to BGR pixels.
/// </summary>
[Benchmark]
public void Rgb24ToBgr24() => PixelConverter.FromRgb24.ToBgr24(this.source3, this.destination3);
/// <summary>
/// Converts BGR pixels to RGBA pixels.
/// </summary>
[Benchmark]
public void Bgr24ToRgba32() => PixelConverter.FromBgr24.ToRgba32(this.source3, this.destination4);
/// <summary>
/// Converts BGR pixels to ARGB pixels.
/// </summary>
[Benchmark]
public void Bgr24ToArgb32() => PixelConverter.FromBgr24.ToArgb32(this.source3, this.destination4);
/// <summary>
/// Converts BGR pixels to ABGR pixels.
/// </summary>
[Benchmark]
public void Bgr24ToAbgr32() => PixelConverter.FromBgr24.ToAbgr32(this.source3, this.destination4);
/// <summary>
/// Converts BGR pixels to BGRA pixels.
/// </summary>
[Benchmark]
public void Bgr24ToBgra32() => PixelConverter.FromBgr24.ToBgra32(this.source3, this.destination4);
/// <summary>
/// Converts BGR pixels to RGB pixels.
/// </summary>
[Benchmark]
public void Bgr24ToRgb24() => PixelConverter.FromBgr24.ToRgb24(this.source3, this.destination3);
}

128
tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs
View File

@ -0,0 +1,128 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion;
/// <summary>
/// Exposes every stateless packed-pixel shuffle operator for assembly inspection.
/// </summary>
/// <remarks>
/// Seven pixels expose the short-input and vector-tail paths. Seventeen pixels execute one
/// full intrinsic group and leave one complete pixel for the scalar remainder, making every
/// part of each generated traversal observable.
/// </remarks>
[Config(typeof(Config.Analysis))]
public class PackedPixelConversionAssembly
{
private byte[] source3;
private byte[] source4;
private byte[] destination3;
private byte[] destination4;
/// <summary>
/// Gets or sets the number of pixels converted by each invocation.
/// </summary>
[Params(7, 17)]
public int Count { get; set; }
/// <summary>
/// Populates the source buffers with deterministic non-uniform channel values.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.source3 = new byte[this.Count * 3];
this.source4 = new byte[this.Count * 4];
this.destination3 = new byte[this.Count * 3];
this.destination4 = new byte[this.Count * 4];
new Random(42).NextBytes(this.source3);
new Random(42).NextBytes(this.source4);
}
/// <summary>
/// Executes the WXYZ four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Wxyz() => SimdUtils.Shuffle4<WXYZShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the WZYX four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Wzyx() => SimdUtils.Shuffle4<WZYXShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the YZWX four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Yzwx() => SimdUtils.Shuffle4<YZWXShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the ZYXW four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Zyxw() => SimdUtils.Shuffle4<ZYXWShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the XWZY four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Xwzy() => SimdUtils.Shuffle4<XWZYShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the XYZ four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Xyz() => SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the YZW four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Yzw() => SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the WZY four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Wzy() => SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the ZYX four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Zyx() => SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the XYZW three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Xyzw() => SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the WXYZ three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Wxyz() => SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the WZYX three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Wzyx() => SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the ZYXW three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Zyxw() => SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the ZYX three-to-three operator.
/// </summary>
[Benchmark]
public void Shuffle3Zyx() => SimdUtils.Shuffle3<ZYXShuffle3>(this.source3, this.destination3);
}

100
tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs
View File

@ -303,50 +303,30 @@ public partial class SimdUtilsTests
{
int size = FeatureTestRunner.Deserialize<int>(serialized);
// These cannot be expressed as a theory as you cannot
// use RemoteExecutor within generic methods nor pass
// IShuffle4 to the generic utils method.
WXYZShuffle4 wxyz = default;
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, wxyz),
(s, d) => SimdUtils.Shuffle4<WXYZShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle2103);
WZYXShuffle4 wzyx = default;
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, wzyx),
(s, d) => SimdUtils.Shuffle4<WZYXShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0123);
YZWXShuffle4 yzwx = default;
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, yzwx),
(s, d) => SimdUtils.Shuffle4<YZWXShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0321);
ZYXWShuffle4 zyxw = default;
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, zyxw),
(s, d) => SimdUtils.Shuffle4<ZYXWShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
DefaultShuffle4 xwyz = new(SimdUtils.Shuffle.MMShuffle2130);
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, xwyz),
xwyz.Control);
DefaultShuffle4 yyyy = new(SimdUtils.Shuffle.MMShuffle1111);
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, yyyy),
yyyy.Control);
DefaultShuffle4 wwww = new(SimdUtils.Shuffle.MMShuffle3333);
TestShuffleByte4Channel(
size,
(s, d) => SimdUtils.Shuffle4(s.Span, d.Span, wwww),
wwww.Control);
(s, d) => SimdUtils.Shuffle4<XWZYShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle1230);
}
FeatureTestRunner.RunWithHwIntrinsicsFeature(
@ -363,32 +343,10 @@ public partial class SimdUtilsTests
{
int size = FeatureTestRunner.Deserialize<int>(serialized);
// These cannot be expressed as a theory as you cannot
// use RemoteExecutor within generic methods nor pass
// IShuffle3 to the generic utils method.
DefaultShuffle3 zyx = new(SimdUtils.Shuffle.MMShuffle3012);
TestShuffleByte3Channel(
size,
(s, d) => SimdUtils.Shuffle3(s.Span, d.Span, zyx),
zyx.Control);
DefaultShuffle3 xyz = new(SimdUtils.Shuffle.MMShuffle3210);
TestShuffleByte3Channel(
size,
(s, d) => SimdUtils.Shuffle3(s.Span, d.Span, xyz),
xyz.Control);
DefaultShuffle3 yyy = new(SimdUtils.Shuffle.MMShuffle3111);
TestShuffleByte3Channel(
size,
(s, d) => SimdUtils.Shuffle3(s.Span, d.Span, yyy),
yyy.Control);
DefaultShuffle3 zzz = new(SimdUtils.Shuffle.MMShuffle3222);
TestShuffleByte3Channel(
size,
(s, d) => SimdUtils.Shuffle3(s.Span, d.Span, zzz),
zzz.Control);
(s, d) => SimdUtils.Shuffle3<ZYXShuffle3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
}
FeatureTestRunner.RunWithHwIntrinsicsFeature(
@ -405,32 +363,25 @@ public partial class SimdUtilsTests
{
int size = FeatureTestRunner.Deserialize<int>(serialized);
// These cannot be expressed as a theory as you cannot
// use RemoteExecutor within generic methods nor pass
// IPad3Shuffle4 to the generic utils method.
XYZWPad3Shuffle4 xyzw = default;
TestPad3Shuffle4Channel(
size,
(s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, xyzw),
(s, d) => SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3210);
DefaultPad3Shuffle4 xwyz = new(SimdUtils.Shuffle.MMShuffle2130);
TestPad3Shuffle4Channel(
size,
(s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, xwyz),
xwyz.Control);
(s, d) => SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle2103);
DefaultPad3Shuffle4 yyyy = new(SimdUtils.Shuffle.MMShuffle1111);
TestPad3Shuffle4Channel(
size,
(s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, yyyy),
yyyy.Control);
(s, d) => SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0123);
DefaultPad3Shuffle4 wwww = new(SimdUtils.Shuffle.MMShuffle3333);
TestPad3Shuffle4Channel(
size,
(s, d) => SimdUtils.Pad3Shuffle4(s.Span, d.Span, wwww),
wwww.Control);
(s, d) => SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
}
FeatureTestRunner.RunWithHwIntrinsicsFeature(
@ -447,32 +398,25 @@ public partial class SimdUtilsTests
{
int size = FeatureTestRunner.Deserialize<int>(serialized);
// These cannot be expressed as a theory as you cannot
// use RemoteExecutor within generic methods nor pass
// IShuffle4Slice3 to the generic utils method.
XYZWShuffle4Slice3 xyzw = default;
TestShuffle4Slice3Channel(
size,
(s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, xyzw),
(s, d) => SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3210);
DefaultShuffle4Slice3 xwyz = new(SimdUtils.Shuffle.MMShuffle2130);
TestShuffle4Slice3Channel(
size,
(s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, xwyz),
xwyz.Control);
(s, d) => SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0321);
DefaultShuffle4Slice3 yyyy = new(SimdUtils.Shuffle.MMShuffle1111);
TestShuffle4Slice3Channel(
size,
(s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, yyyy),
yyyy.Control);
(s, d) => SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0123);
DefaultShuffle4Slice3 wwww = new(SimdUtils.Shuffle.MMShuffle3333);
TestShuffle4Slice3Channel(
size,
(s, d) => SimdUtils.Shuffle4Slice3(s.Span, d.Span, wwww),
wwww.Control);
(s, d) => SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
}
FeatureTestRunner.RunWithHwIntrinsicsFeature(

Write Preview
Loading…
Cancel
Save