mirror of https://github.com/SixLabors/ImageSharp
James Jackson-South
3 years ago
9 changed files with 563 additions and 331 deletions
@ -0,0 +1,178 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Buffers.Binary; |
|||
using System.Diagnostics.CodeAnalysis; |
|||
using System.Numerics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using static SixLabors.ImageSharp.SimdUtils; |
|||
|
|||
namespace SixLabors.ImageSharp; |
|||
|
|||
/// <inheritdoc/>
|
|||
internal interface IShuffle4 : IComponentShuffle |
|||
{ |
|||
} |
|||
|
|||
internal readonly struct DefaultShuffle4([ConstantExpected] byte control) : IShuffle4 |
|||
{ |
|||
public byte Control { get; } = control; |
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination) |
|||
#pragma warning disable CA1857 // A constant is expected for the parameter
|
|||
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, this.Control); |
|||
#pragma warning restore CA1857 // A constant is expected for the parameter
|
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination) |
|||
{ |
|||
ref byte sBase = ref MemoryMarshal.GetReference(source); |
|||
ref byte dBase = ref MemoryMarshal.GetReference(destination); |
|||
|
|||
SimdUtils.Shuffle.InverseMMShuffle(this.Control, out uint p3, out uint p2, out uint p1, out uint p0); |
|||
|
|||
for (nuint i = 0; i < (uint)source.Length; i += 4) |
|||
{ |
|||
Unsafe.Add(ref dBase, i + 0) = Unsafe.Add(ref sBase, p0 + i); |
|||
Unsafe.Add(ref dBase, i + 1) = Unsafe.Add(ref sBase, p1 + i); |
|||
Unsafe.Add(ref dBase, i + 2) = Unsafe.Add(ref sBase, p2 + i); |
|||
Unsafe.Add(ref dBase, i + 3) = Unsafe.Add(ref sBase, p3 + i); |
|||
} |
|||
} |
|||
} |
|||
|
|||
internal readonly struct WXYZShuffle4 : IShuffle4 |
|||
{ |
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination) |
|||
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle2103); |
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination) |
|||
{ |
|||
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source)); |
|||
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination)); |
|||
uint n = (uint)source.Length / 4; |
|||
|
|||
for (nuint i = 0; i < n; i++) |
|||
{ |
|||
uint packed = Unsafe.Add(ref sBase, i); |
|||
|
|||
// packed = [W Z Y X]
|
|||
// ROTL(8, packed) = [Z Y X W]
|
|||
Unsafe.Add(ref dBase, i) = (packed << 8) | (packed >> 24); |
|||
} |
|||
} |
|||
} |
|||
|
|||
internal readonly struct WZYXShuffle4 : IShuffle4 |
|||
{ |
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination) |
|||
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0123); |
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination) |
|||
{ |
|||
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source)); |
|||
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination)); |
|||
uint n = (uint)source.Length / 4; |
|||
|
|||
for (nuint i = 0; i < n; i++) |
|||
{ |
|||
uint packed = Unsafe.Add(ref sBase, i); |
|||
|
|||
// packed = [W Z Y X]
|
|||
// REVERSE(packedArgb) = [X Y Z W]
|
|||
Unsafe.Add(ref dBase, i) = BinaryPrimitives.ReverseEndianness(packed); |
|||
} |
|||
} |
|||
} |
|||
|
|||
internal readonly struct YZWXShuffle4 : IShuffle4 |
|||
{ |
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination) |
|||
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle0321); |
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination) |
|||
{ |
|||
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source)); |
|||
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination)); |
|||
uint n = (uint)source.Length / 4; |
|||
|
|||
for (nuint i = 0; i < n; i++) |
|||
{ |
|||
uint packed = Unsafe.Add(ref sBase, i); |
|||
|
|||
// packed = [W Z Y X]
|
|||
// ROTR(8, packedArgb) = [Y Z W X]
|
|||
Unsafe.Add(ref dBase, i) = BitOperations.RotateRight(packed, 8); |
|||
} |
|||
} |
|||
} |
|||
|
|||
internal readonly struct ZYXWShuffle4 : IShuffle4 |
|||
{ |
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination) |
|||
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle3012); |
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination) |
|||
{ |
|||
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source)); |
|||
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination)); |
|||
uint n = (uint)source.Length / 4; |
|||
|
|||
for (nuint i = 0; i < n; i++) |
|||
{ |
|||
uint packed = Unsafe.Add(ref sBase, i); |
|||
|
|||
// packed = [W Z Y X]
|
|||
// tmp1 = [W 0 Y 0]
|
|||
// tmp2 = [0 Z 0 X]
|
|||
// tmp3=ROTL(16, tmp2) = [0 X 0 Z]
|
|||
// tmp1 + tmp3 = [W X Y Z]
|
|||
uint tmp1 = packed & 0xFF00FF00; |
|||
uint tmp2 = packed & 0x00FF00FF; |
|||
uint tmp3 = BitOperations.RotateLeft(tmp2, 16); |
|||
|
|||
Unsafe.Add(ref dBase, i) = tmp1 + tmp3; |
|||
} |
|||
} |
|||
} |
|||
|
|||
internal readonly struct XWZYShuffle4 : IShuffle4 |
|||
{ |
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> destination) |
|||
=> HwIntrinsics.Shuffle4Reduce(ref source, ref destination, SimdUtils.Shuffle.MMShuffle1230); |
|||
|
|||
[MethodImpl(InliningOptions.ShortMethod)] |
|||
public void Shuffle(ReadOnlySpan<byte> source, Span<byte> destination) |
|||
{ |
|||
ref uint sBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(source)); |
|||
ref uint dBase = ref Unsafe.As<byte, uint>(ref MemoryMarshal.GetReference(destination)); |
|||
uint n = (uint)source.Length / 4; |
|||
|
|||
for (nuint i = 0; i < n; i++) |
|||
{ |
|||
uint packed = Unsafe.Add(ref sBase, i); |
|||
|
|||
// packed = [W Z Y X]
|
|||
// tmp1 = [0 Z 0 X]
|
|||
// tmp2 = [W 0 Y 0]
|
|||
// tmp3=ROTL(16, tmp2) = [Y 0 W 0]
|
|||
// tmp1 + tmp3 = [Y Z W X]
|
|||
uint tmp1 = packed & 0x00FF00FF; |
|||
uint tmp2 = packed & 0xFF00FF00; |
|||
uint tmp3 = BitOperations.RotateLeft(tmp2, 16); |
|||
|
|||
Unsafe.Add(ref dBase, i) = tmp1 + tmp3; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,78 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Diagnostics; |
|||
using System.Diagnostics.CodeAnalysis; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
using System.Runtime.Intrinsics.X86; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
/// <summary>
|
|||
/// Defines utility methods for <see cref="Vector256{T}"/> that have either:
|
|||
/// <list type="number">
|
|||
/// <item>Not yet been normalized in the runtime.</item>
|
|||
/// <item>Produce codegen that is poorly optimized by the runtime.</item>
|
|||
/// </list>
|
|||
/// Should only be used if the intrinsics are available.
|
|||
/// </summary>
|
|||
internal static class Vector256Utilities |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether shuffle byte operations are supported.
|
|||
/// </summary>
|
|||
public static bool SupportsShuffleFloat |
|||
{ |
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
get => Avx.IsSupported; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Gets a value indicating whether shuffle byte operations are supported.
|
|||
/// </summary>
|
|||
public static bool SupportsShuffleByte |
|||
{ |
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
get => Avx2.IsSupported; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Creates a new vector by selecting values from an input vector using a set of indices.
|
|||
/// </summary>
|
|||
/// <param name="vector">The input vector from which values are selected.</param>
|
|||
/// <param name="control">The shuffle control byte.</param>
|
|||
/// <returns>The <see cref="Vector256{Single}"/>.</returns>
|
|||
public static Vector256<float> Shuffle(Vector256<float> vector, [ConstantExpected] byte control) |
|||
{ |
|||
if (Avx.IsSupported) |
|||
{ |
|||
return Avx.Shuffle(vector, vector, control); |
|||
} |
|||
|
|||
ThrowUnreachableException(); |
|||
return default; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Creates a new vector by selecting values from an input vector using a set of indices.</summary>
|
|||
/// <param name="vector">
|
|||
/// The input vector from which values are selected.</param>
|
|||
/// <param name="indices">
|
|||
/// The per-element indices used to select a value from <paramref name="vector" />.
|
|||
/// </param>
|
|||
/// <returns>The <see cref="Vector256{Single}"/>.</returns>
|
|||
public static Vector256<byte> Shuffle(Vector256<byte> vector, Vector256<byte> indices) |
|||
{ |
|||
if (Avx2.IsSupported) |
|||
{ |
|||
return Avx2.Shuffle(vector, indices); |
|||
} |
|||
|
|||
ThrowUnreachableException(); |
|||
return default; |
|||
} |
|||
|
|||
[DoesNotReturn] |
|||
private static void ThrowUnreachableException() => throw new UnreachableException(); |
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Diagnostics; |
|||
using System.Diagnostics.CodeAnalysis; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.Intrinsics; |
|||
using System.Runtime.Intrinsics.X86; |
|||
|
|||
namespace SixLabors.ImageSharp.Common.Helpers; |
|||
|
|||
/// <summary>
|
|||
/// Defines utility methods for <see cref="Vector512{T}"/> that have either:
|
|||
/// <list type="number">
|
|||
/// <item>Not yet been normalized in the runtime.</item>
|
|||
/// <item>Produce codegen that is poorly optimized by the runtime.</item>
|
|||
/// </list>
|
|||
/// Should only be used if the intrinsics are available.
|
|||
/// </summary>
|
|||
internal static class Vector512Utilities |
|||
{ |
|||
/// <summary>
|
|||
/// Gets a value indicating whether shuffle float operations are supported.
|
|||
/// </summary>
|
|||
public static bool SupportsShuffleFloat |
|||
{ |
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
get => Avx512F.IsSupported; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Gets a value indicating whether shuffle byte operations are supported.
|
|||
/// </summary>
|
|||
public static bool SupportsShuffleByte |
|||
{ |
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
get => Avx512BW.IsSupported; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Creates a new vector by selecting values from an input vector using the control.
|
|||
/// </summary>
|
|||
/// <param name="vector">The input vector from which values are selected.</param>
|
|||
/// <param name="control">The shuffle control byte.</param>
|
|||
/// <returns>The <see cref="Vector512{Single}"/>.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<float> Shuffle(Vector512<float> vector, [ConstantExpected] byte control) |
|||
{ |
|||
if (Avx512F.IsSupported) |
|||
{ |
|||
return Avx512F.Shuffle(vector, vector, control); |
|||
} |
|||
|
|||
ThrowUnreachableException(); |
|||
return default; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Creates a new vector by selecting values from an input vector using a set of indices.
|
|||
/// </summary>
|
|||
/// <param name="vector">The input vector from which values are selected.</param>
|
|||
/// <param name="indices">
|
|||
/// The per-element indices used to select a value from <paramref name="vector" />.
|
|||
/// </param>
|
|||
/// <returns>The <see cref="Vector512{Byte}"/>.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
public static Vector512<byte> Shuffle(Vector512<byte> vector, Vector512<byte> indices) |
|||
{ |
|||
if (Avx512BW.IsSupported) |
|||
{ |
|||
return Avx512BW.Shuffle(vector, indices); |
|||
} |
|||
|
|||
ThrowUnreachableException(); |
|||
return default; |
|||
} |
|||
|
|||
[DoesNotReturn] |
|||
private static void ThrowUnreachableException() => throw new UnreachableException(); |
|||
} |
|||
Loading…
Reference in new issue