tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Optimize exact-length RGB plane packing

pull/2633/head
James Jackson-South 1 week ago
parent
dbe310b51f
commit
7bf24aad23
9 changed files with 533 additions and 253 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 185
      src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
  2. 472
      src/ImageSharp/Common/Helpers/SimdUtils.Pack.cs
  3. 13
      src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.RowConverters.cs
  4. 6
      src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs
  5. 22
      src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
  6. 3
      src/ImageSharp/PixelFormats/PixelOperations{TPixel}.cs
  7. 4
      tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs
  8. 64
      tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanesExact.cs
  9. 17
      tests/ImageSharp.Tests/Common/SimdUtilsTests.cs

185
src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
View File

@ -9,13 +9,12 @@ using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.PixelFormats;
namespace SixLabors.ImageSharp;
internal static partial class SimdUtils
{
public static class HwIntrinsics
public static partial class HwIntrinsics
{
#pragma warning disable SA1117 // Parameters should be on same line or separate lines
#pragma warning disable SA1137 // Elements should have the same indentation
@ -25,31 +24,15 @@ internal static partial class SimdUtils
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<int> PermuteMaskDeinterleave16x32() => Vector512.Create(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskEvenOdd8x32() => Vector256.Create(0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskSwitchInnerDWords8x32() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<uint> MoveFirst24BytesToSeparateLanes() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static Vector256<byte> ExtractRgb() => Vector256.Create(0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskPad4Nx16() => Vector128.Create(0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskSlice4Nx16() => Vector128.Create(0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> ShuffleMaskShiftAlpha() => Vector256.Create(
(byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskShiftAlpha8x32() => Vector256.Create(0u, 1, 2, 4, 5, 6, 3, 7);
#pragma warning restore SA1137 // Elements should have the same indentation
#pragma warning restore SA1117 // Parameters should be on same line or separate lines
@ -1038,171 +1021,5 @@ internal static partial class SimdUtils
}
}
}
internal static void PackFromRgbPlanesAvx2Reduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
ref ReadOnlySpan<byte> blueChannel,
ref Span<Rgb24> destination)
{
ref Vector256<byte> rBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(redChannel));
ref Vector256<byte> gBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<byte> bBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(blueChannel));
ref byte dBase = ref Unsafe.As<Rgb24, byte>(ref MemoryMarshal.GetReference(destination));
nuint count = redChannel.Vector256Count<byte>();
Vector256<uint> control1 = PermuteMaskEvenOdd8x32();
Vector256<uint> control2 = PermuteMaskShiftAlpha8x32();
Vector256<byte> a = Vector256.Create((byte)255);
Vector256<byte> shuffleAlpha = ShuffleMaskShiftAlpha();
for (nuint i = 0; i < count; i++)
{
Vector256<byte> r0 = Unsafe.Add(ref rBase, i);
Vector256<byte> g0 = Unsafe.Add(ref gBase, i);
Vector256<byte> b0 = Unsafe.Add(ref bBase, i);
r0 = Avx2.PermuteVar8x32(r0.AsUInt32(), control1).AsByte();
g0 = Avx2.PermuteVar8x32(g0.AsUInt32(), control1).AsByte();
b0 = Avx2.PermuteVar8x32(b0.AsUInt32(), control1).AsByte();
Vector256<byte> rg = Avx2.UnpackLow(r0, g0);
Vector256<byte> b1 = Avx2.UnpackLow(b0, a);
Vector256<byte> rgb1 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte();
Vector256<byte> rgb2 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte();
rg = Avx2.UnpackHigh(r0, g0);
b1 = Avx2.UnpackHigh(b0, a);
Vector256<byte> rgb3 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte();
Vector256<byte> rgb4 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte();
rgb1 = Avx2.Shuffle(rgb1, shuffleAlpha);
rgb2 = Avx2.Shuffle(rgb2, shuffleAlpha);
rgb3 = Avx2.Shuffle(rgb3, shuffleAlpha);
rgb4 = Avx2.Shuffle(rgb4, shuffleAlpha);
rgb1 = Avx2.PermuteVar8x32(rgb1.AsUInt32(), control2).AsByte();
rgb2 = Avx2.PermuteVar8x32(rgb2.AsUInt32(), control2).AsByte();
rgb3 = Avx2.PermuteVar8x32(rgb3.AsUInt32(), control2).AsByte();
rgb4 = Avx2.PermuteVar8x32(rgb4.AsUInt32(), control2).AsByte();
ref byte d1 = ref Unsafe.Add(ref dBase, 24 * 4 * i);
ref byte d2 = ref Unsafe.Add(ref d1, 24);
ref byte d3 = ref Unsafe.Add(ref d2, 24);
ref byte d4 = ref Unsafe.Add(ref d3, 24);
Unsafe.As<byte, Vector256<byte>>(ref d1) = rgb1;
Unsafe.As<byte, Vector256<byte>>(ref d2) = rgb2;
Unsafe.As<byte, Vector256<byte>>(ref d3) = rgb3;
Unsafe.As<byte, Vector256<byte>>(ref d4) = rgb4;
}
int slice = (int)count * Vector256<byte>.Count;
redChannel = redChannel[slice..];
greenChannel = greenChannel[slice..];
blueChannel = blueChannel[slice..];
destination = destination[slice..];
}
internal static void PackFromRgbPlanesAvx2Reduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
ref ReadOnlySpan<byte> blueChannel,
ref Span<Rgba32> destination)
{
ref Vector256<byte> rBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(redChannel));
ref Vector256<byte> gBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<byte> bBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(blueChannel));
ref Vector256<byte> dBase = ref Unsafe.As<Rgba32, Vector256<byte>>(ref MemoryMarshal.GetReference(destination));
nuint count = redChannel.Vector256Count<byte>();
Vector256<uint> control1 = PermuteMaskEvenOdd8x32();
Vector256<byte> a = Vector256.Create((byte)255);
for (nuint i = 0; i < count; i++)
{
Vector256<byte> r0 = Unsafe.Add(ref rBase, i);
Vector256<byte> g0 = Unsafe.Add(ref gBase, i);
Vector256<byte> b0 = Unsafe.Add(ref bBase, i);
r0 = Avx2.PermuteVar8x32(r0.AsUInt32(), control1).AsByte();
g0 = Avx2.PermuteVar8x32(g0.AsUInt32(), control1).AsByte();
b0 = Avx2.PermuteVar8x32(b0.AsUInt32(), control1).AsByte();
Vector256<byte> rg = Avx2.UnpackLow(r0, g0);
Vector256<byte> b1 = Avx2.UnpackLow(b0, a);
Vector256<byte> rgb1 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte();
Vector256<byte> rgb2 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte();
rg = Avx2.UnpackHigh(r0, g0);
b1 = Avx2.UnpackHigh(b0, a);
Vector256<byte> rgb3 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte();
Vector256<byte> rgb4 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte();
ref Vector256<byte> d0 = ref Unsafe.Add(ref dBase, i * 4);
d0 = rgb1;
Unsafe.Add(ref d0, 1) = rgb2;
Unsafe.Add(ref d0, 2) = rgb3;
Unsafe.Add(ref d0, 3) = rgb4;
}
int slice = (int)count * Vector256<byte>.Count;
redChannel = redChannel[slice..];
greenChannel = greenChannel[slice..];
blueChannel = blueChannel[slice..];
destination = destination[slice..];
}
internal static void UnpackToRgbPlanesAvx2Reduce(
ref Span<float> redChannel,
ref Span<float> greenChannel,
ref Span<float> blueChannel,
ref ReadOnlySpan<Rgb24> source)
{
ref Vector256<byte> rgbByteSpan = ref Unsafe.As<Rgb24, Vector256<byte>>(ref MemoryMarshal.GetReference(source));
ref Vector256<float> destRRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(redChannel));
ref Vector256<float> destGRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<float> destBRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(blueChannel));
Vector256<uint> extractToLanesMask = MoveFirst24BytesToSeparateLanes();
Vector256<byte> extractRgbMask = ExtractRgb();
Vector256<byte> rgb, rg, bx;
Vector256<float> r, g, b;
// Each iteration consumes 8 Rgb24 pixels (24 bytes) but starts with a 32-byte load,
// so we need 3 extra pixels of addressable slack beyond the vectorized chunk.
const int bytesPerRgbStride = 24;
nuint count = source.Length > 3 ? (uint)(source.Length - 3) / 8 : 0;
for (nuint i = 0; i < count; i++)
{
rgb = Avx2.PermuteVar8x32(Unsafe.AddByteOffset(ref rgbByteSpan, (uint)(bytesPerRgbStride * i)).AsUInt32(), extractToLanesMask).AsByte();
rgb = Avx2.Shuffle(rgb, extractRgbMask);
rg = Avx2.UnpackLow(rgb, Vector256<byte>.Zero);
bx = Avx2.UnpackHigh(rgb, Vector256<byte>.Zero);
r = Avx.ConvertToVector256Single(Avx2.UnpackLow(rg, Vector256<byte>.Zero).AsInt32());
g = Avx.ConvertToVector256Single(Avx2.UnpackHigh(rg, Vector256<byte>.Zero).AsInt32());
b = Avx.ConvertToVector256Single(Avx2.UnpackLow(bx, Vector256<byte>.Zero).AsInt32());
Unsafe.Add(ref destRRef, i) = r;
Unsafe.Add(ref destGRef, i) = g;
Unsafe.Add(ref destBRef, i) = b;
}
int sliceCount = (int)(count * 8);
redChannel = redChannel[sliceCount..];
greenChannel = greenChannel[sliceCount..];
blueChannel = blueChannel[sliceCount..];
source = source[sliceCount..];
}
}
}

472
src/ImageSharp/Common/Helpers/SimdUtils.Pack.cs
View File

@ -3,7 +3,9 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.PixelFormats;
namespace SixLabors.ImageSharp;
@ -19,11 +21,16 @@ internal static partial class SimdUtils
{
DebugGuard.IsTrue(greenChannel.Length == redChannel.Length, nameof(greenChannel), "Channels must be of same size!");
DebugGuard.IsTrue(blueChannel.Length == redChannel.Length, nameof(blueChannel), "Channels must be of same size!");
DebugGuard.IsTrue(destination.Length > redChannel.Length + 2, nameof(destination), "'destination' must contain a padding of 3 elements!");
DebugGuard.IsTrue(destination.Length >= redChannel.Length, nameof(destination), "'destination' span should not be shorter than the source channels!");
if (Avx2.IsSupported)
{
HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination);
HwIntrinsics.PackFromRgbPlanesReduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination);
}
if (Vector128.IsHardwareAccelerated)
{
PackFromRgbPlanesVector128Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination);
}
else
{
@ -42,11 +49,16 @@ internal static partial class SimdUtils
{
DebugGuard.IsTrue(greenChannel.Length == redChannel.Length, nameof(greenChannel), "Channels must be of same size!");
DebugGuard.IsTrue(blueChannel.Length == redChannel.Length, nameof(blueChannel), "Channels must be of same size!");
DebugGuard.IsTrue(destination.Length > redChannel.Length, nameof(destination), "'destination' span should not be shorter than the source channels!");
DebugGuard.IsTrue(destination.Length >= redChannel.Length, nameof(destination), "'destination' span should not be shorter than the source channels!");
if (Avx2.IsSupported)
{
HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination);
HwIntrinsics.PackFromRgbPlanesReduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination);
}
if (Vector128.IsHardwareAccelerated)
{
PackFromRgbPlanesVector128Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination);
}
else
{
@ -69,12 +81,143 @@ internal static partial class SimdUtils
if (Avx2.IsSupported)
{
HwIntrinsics.UnpackToRgbPlanesAvx2Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref source);
HwIntrinsics.UnpackToRgbPlanesReduce(ref redChannel, ref greenChannel, ref blueChannel, ref source);
}
UnpackToRgbPlanesScalar(redChannel, greenChannel, blueChannel, source);
}
/// <summary>
/// Packs complete sixteen-pixel batches into exact-length <see cref="Rgb24"/> storage using portable 128-bit SIMD.
/// </summary>
/// <param name="redChannel">The red source span, advanced past the converted batches.</param>
/// <param name="greenChannel">The green source span, advanced past the converted batches.</param>
/// <param name="blueChannel">The blue source span, advanced past the converted batches.</param>
/// <param name="destination">The destination span, advanced past the converted batches.</param>
private static void PackFromRgbPlanesVector128Reduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
ref ReadOnlySpan<byte> blueChannel,
ref Span<Rgb24> destination)
{
ref byte redBase = ref MemoryMarshal.GetReference(redChannel);
ref byte greenBase = ref MemoryMarshal.GetReference(greenChannel);
ref byte blueBase = ref MemoryMarshal.GetReference(blueChannel);
ref byte destinationBase = ref Unsafe.As<Rgb24, byte>(ref MemoryMarshal.GetReference(destination));
Vector128<byte> opaqueAlpha = Vector128.Create(byte.MaxValue);
Vector128<byte> removeAlpha = Vector128.Create((byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, byte.MaxValue, byte.MaxValue, byte.MaxValue, byte.MaxValue);
nuint batchCount = (nuint)(uint)redChannel.Length / (uint)Vector128<byte>.Count;
for (nuint i = 0; i < batchCount; i++)
{
nuint sourceOffset = i * (uint)Vector128<byte>.Count;
Vector128<byte> red = Vector128.LoadUnsafe(ref redBase, sourceOffset);
Vector128<byte> green = Vector128.LoadUnsafe(ref greenBase, sourceOffset);
Vector128<byte> blue = Vector128.LoadUnsafe(ref blueBase, sourceOffset);
InterleaveRgbPlanes(red, green, blue, opaqueAlpha, out Vector128<byte> rgba0, out Vector128<byte> rgba1, out Vector128<byte> rgba2, out Vector128<byte> rgba3);
// The native byte shuffle removes alpha from four pixels at a time. Each result owns twelve bytes, so
// exact stores avoid coupling the SIMD path to padding beyond the row or the next memory-group segment.
ref byte destination0 = ref Unsafe.Add(ref destinationBase, i * 48);
StoreRgb24Batch(Vector128.ShuffleNative(rgba0, removeAlpha), ref destination0);
StoreRgb24Batch(Vector128.ShuffleNative(rgba1, removeAlpha), ref Unsafe.Add(ref destination0, 12));
StoreRgb24Batch(Vector128.ShuffleNative(rgba2, removeAlpha), ref Unsafe.Add(ref destination0, 24));
StoreRgb24Batch(Vector128.ShuffleNative(rgba3, removeAlpha), ref Unsafe.Add(ref destination0, 36));
}
int convertedCount = (int)(batchCount * (uint)Vector128<byte>.Count);
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
destination = destination[convertedCount..];
}
/// <summary>
/// Packs complete sixteen-pixel batches into exact-length <see cref="Rgba32"/> storage using portable 128-bit SIMD.
/// </summary>
/// <param name="redChannel">The red source span, advanced past the converted batches.</param>
/// <param name="greenChannel">The green source span, advanced past the converted batches.</param>
/// <param name="blueChannel">The blue source span, advanced past the converted batches.</param>
/// <param name="destination">The destination span, advanced past the converted batches.</param>
private static void PackFromRgbPlanesVector128Reduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
ref ReadOnlySpan<byte> blueChannel,
ref Span<Rgba32> destination)
{
ref byte redBase = ref MemoryMarshal.GetReference(redChannel);
ref byte greenBase = ref MemoryMarshal.GetReference(greenChannel);
ref byte blueBase = ref MemoryMarshal.GetReference(blueChannel);
ref Vector128<byte> destinationBase = ref Unsafe.As<Rgba32, Vector128<byte>>(ref MemoryMarshal.GetReference(destination));
Vector128<byte> opaqueAlpha = Vector128.Create(byte.MaxValue);
nuint batchCount = (nuint)(uint)redChannel.Length / (uint)Vector128<byte>.Count;
for (nuint i = 0; i < batchCount; i++)
{
nuint sourceOffset = i * (uint)Vector128<byte>.Count;
Vector128<byte> red = Vector128.LoadUnsafe(ref redBase, sourceOffset);
Vector128<byte> green = Vector128.LoadUnsafe(ref greenBase, sourceOffset);
Vector128<byte> blue = Vector128.LoadUnsafe(ref blueBase, sourceOffset);
InterleaveRgbPlanes(red, green, blue, opaqueAlpha, out Vector128<byte> rgba0, out Vector128<byte> rgba1, out Vector128<byte> rgba2, out Vector128<byte> rgba3);
ref Vector128<byte> destination0 = ref Unsafe.Add(ref destinationBase, i * 4);
destination0 = rgba0;
Unsafe.Add(ref destination0, 1) = rgba1;
Unsafe.Add(ref destination0, 2) = rgba2;
Unsafe.Add(ref destination0, 3) = rgba3;
}
int convertedCount = (int)(batchCount * (uint)Vector128<byte>.Count);
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
destination = destination[convertedCount..];
}
/// <summary>
/// Interleaves sixteen planar RGB samples into four groups of four opaque RGBA pixels.
/// </summary>
/// <param name="red">The red component lanes.</param>
/// <param name="green">The green component lanes.</param>
/// <param name="blue">The blue component lanes.</param>
/// <param name="alpha">The opaque alpha lanes.</param>
/// <param name="rgba0">The first four interleaved pixels.</param>
/// <param name="rgba1">The second four interleaved pixels.</param>
/// <param name="rgba2">The third four interleaved pixels.</param>
/// <param name="rgba3">The fourth four interleaved pixels.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InterleaveRgbPlanes(
Vector128<byte> red,
Vector128<byte> green,
Vector128<byte> blue,
Vector128<byte> alpha,
out Vector128<byte> rgba0,
out Vector128<byte> rgba1,
out Vector128<byte> rgba2,
out Vector128<byte> rgba3)
{
Vector128<byte> redGreenLow = Vector128_.UnpackLow(red, green);
Vector128<byte> redGreenHigh = Vector128_.UnpackHigh(red, green);
Vector128<byte> blueAlphaLow = Vector128_.UnpackLow(blue, alpha);
Vector128<byte> blueAlphaHigh = Vector128_.UnpackHigh(blue, alpha);
rgba0 = Vector128_.UnpackLow(redGreenLow.AsInt16(), blueAlphaLow.AsInt16()).AsByte();
rgba1 = Vector128_.UnpackHigh(redGreenLow.AsInt16(), blueAlphaLow.AsInt16()).AsByte();
rgba2 = Vector128_.UnpackLow(redGreenHigh.AsInt16(), blueAlphaHigh.AsInt16()).AsByte();
rgba3 = Vector128_.UnpackHigh(redGreenHigh.AsInt16(), blueAlphaHigh.AsInt16()).AsByte();
}
/// <summary>
/// Stores the twelve packed RGB bytes in one shuffled SIMD value without writing its unused lanes.
/// </summary>
/// <param name="value">The packed RGB bytes in the first twelve lanes.</param>
/// <param name="destination">The first destination byte.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void StoreRgb24Batch(Vector128<byte> value, ref byte destination)
{
Unsafe.WriteUnaligned(ref destination, value.AsUInt64().ToScalar());
Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 8), value.AsUInt32().GetElement(2));
}
private static void PackFromRgbPlanesScalarBatchedReduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
@ -86,8 +229,8 @@ internal static partial class SimdUtils
ref ByteTuple4 b = ref Unsafe.As<byte, ByteTuple4>(ref MemoryMarshal.GetReference(blueChannel));
ref Rgb24 rgb = ref MemoryMarshal.GetReference(destination);
nuint count = (uint)redChannel.Length / 4;
for (nuint i = 0; i < count; i++)
nuint batchCount = (uint)redChannel.Length / 4;
for (nuint i = 0; i < batchCount; i++)
{
ref Rgb24 d0 = ref Unsafe.Add(ref rgb, i * 4);
ref Rgb24 d1 = ref Unsafe.Add(ref d0, 1);
@ -115,11 +258,11 @@ internal static partial class SimdUtils
d3.B = bb.V3;
}
int finished = (int)(count * 4);
redChannel = redChannel[finished..];
greenChannel = greenChannel[finished..];
blueChannel = blueChannel[finished..];
destination = destination[finished..];
int convertedCount = (int)(batchCount * 4);
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
destination = destination[convertedCount..];
}
private static void PackFromRgbPlanesScalarBatchedReduce(
@ -133,9 +276,9 @@ internal static partial class SimdUtils
ref ByteTuple4 b = ref Unsafe.As<byte, ByteTuple4>(ref MemoryMarshal.GetReference(blueChannel));
ref Rgba32 rgb = ref MemoryMarshal.GetReference(destination);
nuint count = (uint)redChannel.Length / 4;
nuint batchCount = (uint)redChannel.Length / 4;
destination.Fill(new Rgba32(0, 0, 0, 255));
for (nuint i = 0; i < count; i++)
for (nuint i = 0; i < batchCount; i++)
{
ref Rgba32 d0 = ref Unsafe.Add(ref rgb, i * 4);
ref Rgba32 d1 = ref Unsafe.Add(ref d0, 1);
@ -163,11 +306,11 @@ internal static partial class SimdUtils
d3.B = bb.V3;
}
int finished = (int)(count * 4);
redChannel = redChannel[finished..];
greenChannel = greenChannel[finished..];
blueChannel = blueChannel[finished..];
destination = destination[finished..];
int convertedCount = (int)(batchCount * 4);
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
destination = destination[convertedCount..];
}
private static void PackFromRgbPlanesRemainder(
@ -181,7 +324,7 @@ internal static partial class SimdUtils
ref byte b = ref MemoryMarshal.GetReference(blueChannel);
ref Rgb24 rgb = ref MemoryMarshal.GetReference(destination);
for (nuint i = 0; i < (uint)destination.Length; i++)
for (nuint i = 0; i < (uint)redChannel.Length; i++)
{
ref Rgb24 d = ref Unsafe.Add(ref rgb, i);
d.R = Unsafe.Add(ref r, i);
@ -201,7 +344,7 @@ internal static partial class SimdUtils
ref byte b = ref MemoryMarshal.GetReference(blueChannel);
ref Rgba32 rgba = ref MemoryMarshal.GetReference(destination);
for (nuint i = 0; i < (uint)destination.Length; i++)
for (nuint i = 0; i < (uint)redChannel.Length; i++)
{
ref Rgba32 d = ref Unsafe.Add(ref rgba, i);
d.R = Unsafe.Add(ref r, i);
@ -234,4 +377,291 @@ internal static partial class SimdUtils
Unsafe.Add(ref b, i) = src.B;
}
}
/// <summary>
/// Provides the hardware-intrinsic reducers used by the planar RGB packing pipeline.
/// </summary>
public static partial class HwIntrinsics
{
/// <summary>
/// Creates the AVX2 lane order used before interleaving planar RGB components.
/// </summary>
/// <returns>The source lane permutation.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskEvenOdd8x32() => Vector256.Create(0u, 2, 4, 6, 1, 3, 5, 7);
/// <summary>
/// Packs complete AVX2 batches into <see cref="Rgb24"/> pixels and retains the unconverted remainder.
/// </summary>
/// <param name="redChannel">The red source span.</param>
/// <param name="greenChannel">The green source span.</param>
/// <param name="blueChannel">The blue source span.</param>
/// <param name="destination">The destination pixel span.</param>
internal static void PackFromRgbPlanesReduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
ref ReadOnlySpan<byte> blueChannel,
ref Span<Rgb24> destination)
{
ref Vector256<byte> redBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(redChannel));
ref Vector256<byte> greenBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<byte> blueBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(blueChannel));
ref byte destinationBase = ref Unsafe.As<Rgb24, byte>(ref MemoryMarshal.GetReference(destination));
nuint batchCount = redChannel.Vector256Count<byte>();
Vector256<uint> sourceOrder = PermuteMaskEvenOdd8x32();
Vector256<uint> packedOrder = Vector256.Create(0u, 1, 2, 4, 5, 6, 3, 7);
Vector256<byte> opaqueAlpha = Vector256.Create(byte.MaxValue);
Vector128<byte> removeAlphaLower = Vector128.Create((byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15);
Vector128<byte> removeAlphaUpper = Vector128.Create((byte)16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30, 19, 23, 27, 31);
Vector256<byte> removeAlpha = Vector256.Create(removeAlphaLower, removeAlphaUpper);
bool hasWritablePadding = destination.Length >= redChannel.Length + 3;
nuint i = 0;
// Non-final batches retain the original four overlapping wide stores. Splitting the final batch keeps
// the exact-row decision out of the hot loop and limits the narrower stores to the only bytes that can
// cross the destination boundary.
for (; i + 1 < batchCount; i++)
{
PackRgb24Batch(
Unsafe.Add(ref redBase, i),
Unsafe.Add(ref greenBase, i),
Unsafe.Add(ref blueBase, i),
opaqueAlpha,
sourceOrder,
packedOrder,
removeAlpha,
out Vector256<byte> rgb0,
out Vector256<byte> rgb1,
out Vector256<byte> rgb2,
out Vector256<byte> rgb3);
ref byte destination0 = ref Unsafe.Add(ref destinationBase, 96 * i);
ref byte destination1 = ref Unsafe.Add(ref destination0, 24);
ref byte destination2 = ref Unsafe.Add(ref destination1, 24);
ref byte destination3 = ref Unsafe.Add(ref destination2, 24);
Unsafe.As<byte, Vector256<byte>>(ref destination0) = rgb0;
Unsafe.As<byte, Vector256<byte>>(ref destination1) = rgb1;
Unsafe.As<byte, Vector256<byte>>(ref destination2) = rgb2;
Unsafe.As<byte, Vector256<byte>>(ref destination3) = rgb3;
}
if (i < batchCount)
{
PackRgb24Batch(
Unsafe.Add(ref redBase, i),
Unsafe.Add(ref greenBase, i),
Unsafe.Add(ref blueBase, i),
opaqueAlpha,
sourceOrder,
packedOrder,
removeAlpha,
out Vector256<byte> rgb0,
out Vector256<byte> rgb1,
out Vector256<byte> rgb2,
out Vector256<byte> rgb3);
ref byte destination0 = ref Unsafe.Add(ref destinationBase, 96 * i);
ref byte destination1 = ref Unsafe.Add(ref destination0, 24);
ref byte destination2 = ref Unsafe.Add(ref destination1, 24);
ref byte destination3 = ref Unsafe.Add(ref destination2, 24);
Unsafe.As<byte, Vector256<byte>>(ref destination0) = rgb0;
Unsafe.As<byte, Vector256<byte>>(ref destination1) = rgb1;
Unsafe.As<byte, Vector256<byte>>(ref destination2) = rgb2;
if (hasWritablePadding)
{
Unsafe.As<byte, Vector256<byte>>(ref destination3) = rgb3;
}
else
{
// The final compacted vector contains 24 RGB bytes followed by eight unused bytes. Exact stores
// retain all useful bytes without writing beyond an unpadded destination row.
Unsafe.As<byte, Vector128<byte>>(ref destination3) = rgb3.GetLower();
Unsafe.As<byte, ulong>(ref Unsafe.Add(ref destination3, 16)) = rgb3.GetUpper().AsUInt64().ToScalar();
}
}
int convertedCount = (int)batchCount * Vector256<byte>.Count;
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
destination = destination[convertedCount..];
}
/// <summary>
/// Packs complete AVX2 batches into <see cref="Rgba32"/> pixels and retains the unconverted remainder.
/// </summary>
/// <param name="redChannel">The red source span.</param>
/// <param name="greenChannel">The green source span.</param>
/// <param name="blueChannel">The blue source span.</param>
/// <param name="destination">The destination pixel span.</param>
internal static void PackFromRgbPlanesReduce(
ref ReadOnlySpan<byte> redChannel,
ref ReadOnlySpan<byte> greenChannel,
ref ReadOnlySpan<byte> blueChannel,
ref Span<Rgba32> destination)
{
ref Vector256<byte> redBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(redChannel));
ref Vector256<byte> greenBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<byte> blueBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(blueChannel));
ref Vector256<byte> destinationBase = ref Unsafe.As<Rgba32, Vector256<byte>>(ref MemoryMarshal.GetReference(destination));
nuint batchCount = redChannel.Vector256Count<byte>();
Vector256<uint> sourceOrder = PermuteMaskEvenOdd8x32();
Vector256<byte> opaqueAlpha = Vector256.Create(byte.MaxValue);
for (nuint i = 0; i < batchCount; i++)
{
InterleaveRgbPlanes(
Unsafe.Add(ref redBase, i),
Unsafe.Add(ref greenBase, i),
Unsafe.Add(ref blueBase, i),
opaqueAlpha,
sourceOrder,
out Vector256<byte> rgba0,
out Vector256<byte> rgba1,
out Vector256<byte> rgba2,
out Vector256<byte> rgba3);
ref Vector256<byte> destination0 = ref Unsafe.Add(ref destinationBase, i * 4);
destination0 = rgba0;
Unsafe.Add(ref destination0, 1) = rgba1;
Unsafe.Add(ref destination0, 2) = rgba2;
Unsafe.Add(ref destination0, 3) = rgba3;
}
int convertedCount = (int)batchCount * Vector256<byte>.Count;
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
destination = destination[convertedCount..];
}
/// <summary>
/// Unpacks complete AVX2 batches from <see cref="Rgb24"/> pixels and retains the unconverted remainder.
/// </summary>
/// <param name="redChannel">The red destination span.</param>
/// <param name="greenChannel">The green destination span.</param>
/// <param name="blueChannel">The blue destination span.</param>
/// <param name="source">The source pixel span.</param>
internal static void UnpackToRgbPlanesReduce(
ref Span<float> redChannel,
ref Span<float> greenChannel,
ref Span<float> blueChannel,
ref ReadOnlySpan<Rgb24> source)
{
ref Vector256<byte> sourceBase = ref Unsafe.As<Rgb24, Vector256<byte>>(ref MemoryMarshal.GetReference(source));
ref Vector256<float> redBase = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(redChannel));
ref Vector256<float> greenBase = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<float> blueBase = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(blueChannel));
Vector256<uint> separateLanes = Vector256.Create(0u, 1, 2, 6, 3, 4, 5, 7);
Vector128<byte> extractRgbLower = Vector128.Create((byte)0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, byte.MaxValue, byte.MaxValue, byte.MaxValue, byte.MaxValue);
Vector128<byte> extractRgbUpper = Vector128.Create((byte)16, 19, 22, 25, 17, 20, 23, 26, 18, 21, 24, 27, byte.MaxValue, byte.MaxValue, byte.MaxValue, byte.MaxValue);
Vector256<byte> extractRgb = Vector256.Create(extractRgbLower, extractRgbUpper);
// Each iteration consumes eight Rgb24 pixels, or 24 bytes, but starts with a 32-byte load. Three extra
// source pixels must therefore remain addressable beyond every vectorized batch.
const int bytesPerBatch = 24;
nuint batchCount = source.Length > 3 ? (uint)(source.Length - 3) / 8 : 0;
for (nuint i = 0; i < batchCount; i++)
{
Vector256<byte> packed = Avx2.PermuteVar8x32(Unsafe.AddByteOffset(ref sourceBase, (uint)(bytesPerBatch * i)).AsUInt32(), separateLanes).AsByte();
packed = Vector256.ShuffleNative(packed, extractRgb);
Vector256<byte> redGreen = Avx2.UnpackLow(packed, Vector256<byte>.Zero);
Vector256<byte> blue = Avx2.UnpackHigh(packed, Vector256<byte>.Zero);
Vector256<float> red = Avx.ConvertToVector256Single(Avx2.UnpackLow(redGreen, Vector256<byte>.Zero).AsInt32());
Vector256<float> green = Avx.ConvertToVector256Single(Avx2.UnpackHigh(redGreen, Vector256<byte>.Zero).AsInt32());
Vector256<float> blueValues = Avx.ConvertToVector256Single(Avx2.UnpackLow(blue, Vector256<byte>.Zero).AsInt32());
Unsafe.Add(ref redBase, i) = red;
Unsafe.Add(ref greenBase, i) = green;
Unsafe.Add(ref blueBase, i) = blueValues;
}
int convertedCount = (int)(batchCount * 8);
redChannel = redChannel[convertedCount..];
greenChannel = greenChannel[convertedCount..];
blueChannel = blueChannel[convertedCount..];
source = source[convertedCount..];
}
/// <summary>
/// Interleaves and compacts one AVX2 batch into four groups of eight <see cref="Rgb24"/> pixels.
/// </summary>
/// <param name="red">The red component lanes.</param>
/// <param name="green">The green component lanes.</param>
/// <param name="blue">The blue component lanes.</param>
/// <param name="alpha">The opaque alpha lanes used during interleaving.</param>
/// <param name="sourceOrder">The cross-lane source permutation.</param>
/// <param name="packedOrder">The cross-lane packed RGB permutation.</param>
/// <param name="removeAlpha">The native byte-shuffle indices that compact RGBA to RGB.</param>
/// <param name="rgb0">The first eight packed pixels.</param>
/// <param name="rgb1">The second eight packed pixels.</param>
/// <param name="rgb2">The third eight packed pixels.</param>
/// <param name="rgb3">The fourth eight packed pixels.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void PackRgb24Batch(
Vector256<byte> red,
Vector256<byte> green,
Vector256<byte> blue,
Vector256<byte> alpha,
Vector256<uint> sourceOrder,
Vector256<uint> packedOrder,
Vector256<byte> removeAlpha,
out Vector256<byte> rgb0,
out Vector256<byte> rgb1,
out Vector256<byte> rgb2,
out Vector256<byte> rgb3)
{
InterleaveRgbPlanes(red, green, blue, alpha, sourceOrder, out Vector256<byte> rgba0, out Vector256<byte> rgba1, out Vector256<byte> rgba2, out Vector256<byte> rgba3);
rgb0 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba0, removeAlpha).AsUInt32(), packedOrder).AsByte();
rgb1 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba1, removeAlpha).AsUInt32(), packedOrder).AsByte();
rgb2 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba2, removeAlpha).AsUInt32(), packedOrder).AsByte();
rgb3 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba3, removeAlpha).AsUInt32(), packedOrder).AsByte();
}
/// <summary>
/// Interleaves 32 planar RGB samples into four groups of eight opaque RGBA pixels.
/// </summary>
/// <param name="red">The red component lanes.</param>
/// <param name="green">The green component lanes.</param>
/// <param name="blue">The blue component lanes.</param>
/// <param name="alpha">The opaque alpha lanes.</param>
/// <param name="sourceOrder">The cross-lane source permutation.</param>
/// <param name="rgba0">The first eight interleaved pixels.</param>
/// <param name="rgba1">The second eight interleaved pixels.</param>
/// <param name="rgba2">The third eight interleaved pixels.</param>
/// <param name="rgba3">The fourth eight interleaved pixels.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InterleaveRgbPlanes(
Vector256<byte> red,
Vector256<byte> green,
Vector256<byte> blue,
Vector256<byte> alpha,
Vector256<uint> sourceOrder,
out Vector256<byte> rgba0,
out Vector256<byte> rgba1,
out Vector256<byte> rgba2,
out Vector256<byte> rgba3)
{
red = Avx2.PermuteVar8x32(red.AsUInt32(), sourceOrder).AsByte();
green = Avx2.PermuteVar8x32(green.AsUInt32(), sourceOrder).AsByte();
blue = Avx2.PermuteVar8x32(blue.AsUInt32(), sourceOrder).AsByte();
Vector256<byte> redGreenLow = Avx2.UnpackLow(red, green);
Vector256<byte> redGreenHigh = Avx2.UnpackHigh(red, green);
Vector256<byte> blueAlphaLow = Avx2.UnpackLow(blue, alpha);
Vector256<byte> blueAlphaHigh = Avx2.UnpackHigh(blue, alpha);
rgba0 = Avx2.UnpackLow(redGreenLow.AsUInt16(), blueAlphaLow.AsUInt16()).AsByte();
rgba1 = Avx2.UnpackHigh(redGreenLow.AsUInt16(), blueAlphaLow.AsUInt16()).AsByte();
rgba2 = Avx2.UnpackLow(redGreenHigh.AsUInt16(), blueAlphaHigh.AsUInt16()).AsByte();
rgba3 = Avx2.UnpackHigh(redGreenHigh.AsUInt16(), blueAlphaHigh.AsUInt16()).AsByte();
}
}
}

13
src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.RowConverters.cs
View File

@ -129,8 +129,7 @@ internal static partial class Av1YuvConverter
/// </summary>
/// <param name="y">The row index.</param>
/// <param name="span">The reusable conversion buffer.</param>
/// <param name="proxy">The padded destination used when an eight-bit image row cannot expose sufficient padding.</param>
public void Convert(int y, Span<float> span, Span<TPixel> proxy)
public void Convert(int y, Span<float> span)
{
int width = this.image.Width;
Span<float> red = span[..width];
@ -203,15 +202,7 @@ internal static partial class Av1YuvConverter
SimdUtils.NormalizedFloatToByteSaturate(green, greenBytes);
SimdUtils.NormalizedFloatToByteSaturate(blue, blueBytes);
if (this.image.PixelBuffer.DangerousTryGetPaddedRowSpan(y, 3, out Span<TPixel> paddedDestination))
{
PixelOperations<TPixel>.Instance.PackFromRgbPlanes(redBytes, greenBytes, blueBytes, paddedDestination);
}
else
{
PixelOperations<TPixel>.Instance.PackFromRgbPlanes(redBytes, greenBytes, blueBytes, proxy);
proxy[..width].CopyTo(destination);
}
PixelOperations<TPixel>.Instance.PackFromRgbPlanes(redBytes, greenBytes, blueBytes, destination);
}
else
{

6
src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs
View File

@ -44,12 +44,10 @@ internal static partial class Av1YuvConverter
{
YuvToRgbRowConverter<TPixel, byte, HeifByteSampleLoader> converter = new(configuration, frameBuffer, image, colorConverter);
using IMemoryOwner<float> scratchOwner = configuration.MemoryAllocator.Allocate<float>(converter.BufferLength);
using IMemoryOwner<TPixel> proxyOwner = configuration.MemoryAllocator.Allocate<TPixel>(image.Width + 3);
Span<float> scratch = scratchOwner.GetSpan();
Span<TPixel> proxy = proxyOwner.GetSpan()[..(image.Width + 3)];
for (int y = 0; y < image.Height; y++)
{
converter.Convert(y, scratch, proxy);
converter.Convert(y, scratch);
}
}
else
@ -59,7 +57,7 @@ internal static partial class Av1YuvConverter
Span<float> scratch = owner.GetSpan();
for (int y = 0; y < image.Height; y++)
{
converter.Convert(y, scratch, Span<TPixel>.Empty);
converter.Convert(y, scratch);
}
}
}

22
src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
View File

@ -42,11 +42,6 @@ internal class SpectralConverter<TPixel> : SpectralConverter, IDisposable
/// </summary>
private IMemoryOwner<byte> rgbBuffer;
/// <summary>
/// Proxy buffer used in packing from RGB to target TPixel pixels.
/// </summary>
private IMemoryOwner<TPixel> paddedProxyPixelRow;
/// <summary>
/// Resulting 2D pixel buffer.
/// </summary>
@ -163,19 +158,8 @@ internal class SpectralConverter<TPixel> : SpectralConverter, IDisposable
SimdUtils.NormalizedFloatToByteSaturate(values.Component1, g);
SimdUtils.NormalizedFloatToByteSaturate(values.Component2, b);
// PackFromRgbPlanes expects the destination to be padded, so try to get padded span containing extra elements from the next row.
// If we can't get such a padded row because we are on a MemoryGroup boundary or at the last row,
// pack pixels to a temporary, padded proxy buffer, then copy the relevant values to the destination row.
if (this.pixelBuffer.DangerousTryGetPaddedRowSpan(yy, 3, out Span<TPixel> destRow))
{
PixelOperations<TPixel>.Instance.PackFromRgbPlanes(r, g, b, destRow);
}
else
{
Span<TPixel> proxyRow = this.paddedProxyPixelRow.GetSpan();
PixelOperations<TPixel>.Instance.PackFromRgbPlanes(r, g, b, proxyRow);
proxyRow[..width].CopyTo(this.pixelBuffer.DangerousGetRowSpan(yy));
}
Span<TPixel> destination = this.pixelBuffer.DangerousGetRowSpan(yy);
PixelOperations<TPixel>.Instance.PackFromRgbPlanes(r, g, b, destination);
}
this.pixelRowCounter += this.pixelRowsPerStep;
@ -214,7 +198,6 @@ internal class SpectralConverter<TPixel> : SpectralConverter, IDisposable
pixelSize.Height,
this.Configuration.PreferContiguousImageBuffers,
AllocationOptions.Clean);
this.paddedProxyPixelRow = allocator.Allocate<TPixel>(pixelSize.Width + 3);
// Component processors from spectral to RGB
int bufferWidth = majorBlockWidth * blockPixelSize;
@ -275,7 +258,6 @@ internal class SpectralConverter<TPixel> : SpectralConverter, IDisposable
}
this.rgbBuffer?.Dispose();
this.paddedProxyPixelRow?.Dispose();
this.pixelBuffer?.Dispose();
}
}

3
src/ImageSharp/PixelFormats/PixelOperations{TPixel}.cs
View File

@ -348,7 +348,6 @@ public partial class PixelOperations<TPixel>
/// <summary>
/// Bulk operation that packs 3 separate RGB channels to <paramref name="destination"/>.
/// The destination must have a padding of 3.
/// </summary>
/// <param name="redChannel">A <see cref="ReadOnlySpan{T}"/> to the red values.</param>
/// <param name="greenChannel">A <see cref="ReadOnlySpan{T}"/> to the green values.</param>
@ -424,6 +423,6 @@ public partial class PixelOperations<TPixel>
{
Guard.IsTrue(greenChannel.Length == count, nameof(greenChannel), "Channels must be of same size!");
Guard.IsTrue(blueChannel.Length == count, nameof(blueChannel), "Channels must be of same size!");
Guard.IsTrue(destination.Length > count + 2, nameof(destination), "'destination' must contain a padding of 3 elements!");
Guard.IsTrue(destination.Length >= count, nameof(destination), "'destination' span should not be shorter than the source channels!");
}
}

4
tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs
View File

@ -244,7 +244,7 @@ public unsafe class PixelConversion_PackFromRgbPlanes
ReadOnlySpan<byte> g = this.rBuf;
ReadOnlySpan<byte> b = this.rBuf;
Span<Rgb24> rgb = this.rgbBuf;
SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref r, ref g, ref b, ref rgb);
SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref r, ref g, ref b, ref rgb);
}
[Benchmark]
@ -254,7 +254,7 @@ public unsafe class PixelConversion_PackFromRgbPlanes
ReadOnlySpan<byte> g = this.rBuf;
ReadOnlySpan<byte> b = this.rBuf;
Span<Rgba32> rgb = this.rgbaBuf;
SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref r, ref g, ref b, ref rgb);
SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref r, ref g, ref b, ref rgb);
}
#pragma warning disable SA1132

64
tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanesExact.cs
View File

@ -0,0 +1,64 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.PixelFormats;
namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion;
/// <summary>
/// Measures planar RGB packing for padded and exact-length decoder rows.
/// </summary>
public class PixelConversion_PackFromRgbPlanesExact
{
private byte[] red;
private byte[] green;
private byte[] blue;
private Rgb24[] exactDestination;
private Rgb24[] paddedDestination;
/// <summary>
/// Gets or sets the decoded row width.
/// </summary>
[Params(1920, 4242)]
public int Count { get; set; }
/// <summary>
/// Creates deterministic component planes and destination rows outside the measured operation.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.red = new byte[this.Count];
this.green = new byte[this.Count];
this.blue = new byte[this.Count];
this.exactDestination = new Rgb24[this.Count];
this.paddedDestination = new Rgb24[this.Count + 3];
new Random(42).NextBytes(this.red);
new Random(43).NextBytes(this.green);
new Random(44).NextBytes(this.blue);
}
/// <summary>
/// Packs a row using the legacy decoder contract with three writable destination pixels beyond the row.
/// </summary>
[Benchmark(Baseline = true)]
public void PaddedDestination() => SimdUtils.PackFromRgbPlanes(this.red, this.green, this.blue, this.paddedDestination);
/// <summary>
/// Packs through a padded proxy and copies the completed row, matching the former decoder fallback.
/// </summary>
[Benchmark]
public void PaddedProxyAndCopy()
{
SimdUtils.PackFromRgbPlanes(this.red, this.green, this.blue, this.paddedDestination);
this.paddedDestination.AsSpan(0, this.Count).CopyTo(this.exactDestination);
}
/// <summary>
/// Packs a row directly into the exact-length destination exposed by an image frame.
/// </summary>
[Benchmark]
public void ExactDestination() => SimdUtils.PackFromRgbPlanes(this.red, this.green, this.blue, this.exactDestination);
}

17
tests/ImageSharp.Tests/Common/SimdUtilsTests.cs
View File

@ -272,7 +272,7 @@ public partial class SimdUtilsTests
SimdUtils.PackFromRgbPlanes(r, g, b, actual));
[Fact]
public void PackFromRgbPlanesAvx2Reduce_Rgb24()
public void PackFromRgbPlanesReduce_Rgb24()
{
if (!Avx2.IsSupported)
{
@ -282,15 +282,14 @@ public partial class SimdUtilsTests
byte[] r = [.. Enumerable.Range(0, 32).Select(x => (byte)x)];
byte[] g = [.. Enumerable.Range(100, 32).Select(x => (byte)x)];
byte[] b = [.. Enumerable.Range(200, 32).Select(x => (byte)x)];
const int padding = 4;
Rgb24[] d = new Rgb24[32 + padding];
Rgb24[] d = new Rgb24[32];
ReadOnlySpan<byte> rr = r.AsSpan();
ReadOnlySpan<byte> gg = g.AsSpan();
ReadOnlySpan<byte> bb = b.AsSpan();
Span<Rgb24> dd = d.AsSpan();
SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref rr, ref gg, ref bb, ref dd);
SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref rr, ref gg, ref bb, ref dd);
for (int i = 0; i < 32; i++)
{
@ -302,11 +301,11 @@ public partial class SimdUtilsTests
Assert.Equal(0, rr.Length);
Assert.Equal(0, gg.Length);
Assert.Equal(0, bb.Length);
Assert.Equal(padding, dd.Length);
Assert.Equal(0, dd.Length);
}
[Fact]
public void PackFromRgbPlanesAvx2Reduce_Rgba32()
public void PackFromRgbPlanesReduce_Rgba32()
{
if (!Avx2.IsSupported)
{
@ -324,7 +323,7 @@ public partial class SimdUtilsTests
ReadOnlySpan<byte> bb = b.AsSpan();
Span<Rgba32> dd = d.AsSpan();
SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref rr, ref gg, ref bb, ref dd);
SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref rr, ref gg, ref bb, ref dd);
for (int i = 0; i < 32; i++)
{
@ -354,10 +353,10 @@ public partial class SimdUtilsTests
expected[i] = TPixel.FromRgb24(new Rgb24(r[i], g[i], b[i]));
}
TPixel[] actual = new TPixel[count + 3]; // padding for Rgb24 AVX2
TPixel[] actual = new TPixel[count];
packMethod(r, g, b, actual);
Assert.True(expected.AsSpan().SequenceEqual(actual.AsSpan()[..count]));
Assert.True(expected.AsSpan().SequenceEqual(actual));
}
private static void TestImpl_BulkConvertNormalizedFloatToByteClampOverflows(

Write Preview
Loading…
Cancel
Save