From 7bf24aad236c593dcd8d8d9043742d99dda5aea6 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 26 Aug 2026 10:13:50 +1000 Subject: [PATCH] Optimize exact-length RGB plane packing --- .../Common/Helpers/SimdUtils.HwIntrinsics.cs | 185 +------ .../Common/Helpers/SimdUtils.Pack.cs | 472 +++++++++++++++++- .../Heif/Av1/Av1YuvConverter.RowConverters.cs | 13 +- .../Formats/Heif/Av1/Av1YuvConverter.cs | 6 +- .../Decoder/SpectralConverter{TPixel}.cs | 22 +- .../PixelFormats/PixelOperations{TPixel}.cs | 3 +- .../PixelConversion_PackFromRgbPlanes.cs | 4 +- .../PixelConversion_PackFromRgbPlanesExact.cs | 64 +++ .../ImageSharp.Tests/Common/SimdUtilsTests.cs | 17 +- 9 files changed, 533 insertions(+), 253 deletions(-) create mode 100644 tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanesExact.cs diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs b/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs index 1740df5b5..c3b934b18 100644 --- a/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs +++ b/src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs @@ -9,13 +9,12 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Common.Helpers; -using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp; internal static partial class SimdUtils { - public static class HwIntrinsics + public static partial class HwIntrinsics { #pragma warning disable SA1117 // Parameters should be on same line or separate lines #pragma warning disable SA1137 // Elements should have the same indentation @@ -25,31 +24,15 @@ internal static partial class SimdUtils [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector512 PermuteMaskDeinterleave16x32() => Vector512.Create(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 PermuteMaskEvenOdd8x32() => Vector256.Create(0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32(); - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector256 PermuteMaskSwitchInnerDWords8x32() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0).AsUInt32(); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 MoveFirst24BytesToSeparateLanes() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static Vector256 ExtractRgb() => Vector256.Create(0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF); - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 ShuffleMaskPad4Nx16() => Vector128.Create(0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80); [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 ShuffleMaskSlice4Nx16() => Vector128.Create(0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 ShuffleMaskShiftAlpha() => Vector256.Create( - (byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15, - 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 PermuteMaskShiftAlpha8x32() => Vector256.Create(0u, 1, 2, 4, 5, 6, 3, 7); #pragma warning restore SA1137 // Elements should have the same indentation #pragma warning restore SA1117 // Parameters should be on same line or separate lines @@ -1038,171 +1021,5 @@ internal static partial class SimdUtils } } } - - internal static void PackFromRgbPlanesAvx2Reduce( - ref ReadOnlySpan redChannel, - ref ReadOnlySpan greenChannel, - ref ReadOnlySpan blueChannel, - ref Span destination) - { - ref Vector256 rBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(redChannel)); - ref Vector256 gBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(greenChannel)); - ref Vector256 bBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(blueChannel)); - ref byte dBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); - - nuint count = redChannel.Vector256Count(); - - Vector256 control1 = PermuteMaskEvenOdd8x32(); - - Vector256 control2 = PermuteMaskShiftAlpha8x32(); - Vector256 a = Vector256.Create((byte)255); - - Vector256 shuffleAlpha = ShuffleMaskShiftAlpha(); - - for (nuint i = 0; i < count; i++) - { - Vector256 r0 = Unsafe.Add(ref rBase, i); - Vector256 g0 = Unsafe.Add(ref gBase, i); - Vector256 b0 = Unsafe.Add(ref bBase, i); - - r0 = Avx2.PermuteVar8x32(r0.AsUInt32(), control1).AsByte(); - g0 = Avx2.PermuteVar8x32(g0.AsUInt32(), control1).AsByte(); - b0 = Avx2.PermuteVar8x32(b0.AsUInt32(), control1).AsByte(); - - Vector256 rg = Avx2.UnpackLow(r0, g0); - Vector256 b1 = Avx2.UnpackLow(b0, a); - - Vector256 rgb1 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - Vector256 rgb2 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - - rg = Avx2.UnpackHigh(r0, g0); - b1 = Avx2.UnpackHigh(b0, a); - - Vector256 rgb3 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - Vector256 rgb4 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - - rgb1 = Avx2.Shuffle(rgb1, shuffleAlpha); - rgb2 = Avx2.Shuffle(rgb2, shuffleAlpha); - rgb3 = Avx2.Shuffle(rgb3, shuffleAlpha); - rgb4 = Avx2.Shuffle(rgb4, shuffleAlpha); - - rgb1 = Avx2.PermuteVar8x32(rgb1.AsUInt32(), control2).AsByte(); - rgb2 = Avx2.PermuteVar8x32(rgb2.AsUInt32(), control2).AsByte(); - rgb3 = Avx2.PermuteVar8x32(rgb3.AsUInt32(), control2).AsByte(); - rgb4 = Avx2.PermuteVar8x32(rgb4.AsUInt32(), control2).AsByte(); - - ref byte d1 = ref Unsafe.Add(ref dBase, 24 * 4 * i); - ref byte d2 = ref Unsafe.Add(ref d1, 24); - ref byte d3 = ref Unsafe.Add(ref d2, 24); - ref byte d4 = ref Unsafe.Add(ref d3, 24); - - Unsafe.As>(ref d1) = rgb1; - Unsafe.As>(ref d2) = rgb2; - Unsafe.As>(ref d3) = rgb3; - Unsafe.As>(ref d4) = rgb4; - } - - int slice = (int)count * Vector256.Count; - redChannel = redChannel[slice..]; - greenChannel = greenChannel[slice..]; - blueChannel = blueChannel[slice..]; - destination = destination[slice..]; - } - - internal static void PackFromRgbPlanesAvx2Reduce( - ref ReadOnlySpan redChannel, - ref ReadOnlySpan greenChannel, - ref ReadOnlySpan blueChannel, - ref Span destination) - { - ref Vector256 rBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(redChannel)); - ref Vector256 gBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(greenChannel)); - ref Vector256 bBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(blueChannel)); - ref Vector256 dBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); - - nuint count = redChannel.Vector256Count(); - Vector256 control1 = PermuteMaskEvenOdd8x32(); - Vector256 a = Vector256.Create((byte)255); - - for (nuint i = 0; i < count; i++) - { - Vector256 r0 = Unsafe.Add(ref rBase, i); - Vector256 g0 = Unsafe.Add(ref gBase, i); - Vector256 b0 = Unsafe.Add(ref bBase, i); - - r0 = Avx2.PermuteVar8x32(r0.AsUInt32(), control1).AsByte(); - g0 = Avx2.PermuteVar8x32(g0.AsUInt32(), control1).AsByte(); - b0 = Avx2.PermuteVar8x32(b0.AsUInt32(), control1).AsByte(); - - Vector256 rg = Avx2.UnpackLow(r0, g0); - Vector256 b1 = Avx2.UnpackLow(b0, a); - - Vector256 rgb1 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - Vector256 rgb2 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - - rg = Avx2.UnpackHigh(r0, g0); - b1 = Avx2.UnpackHigh(b0, a); - - Vector256 rgb3 = Avx2.UnpackLow(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - Vector256 rgb4 = Avx2.UnpackHigh(rg.AsUInt16(), b1.AsUInt16()).AsByte(); - - ref Vector256 d0 = ref Unsafe.Add(ref dBase, i * 4); - d0 = rgb1; - Unsafe.Add(ref d0, 1) = rgb2; - Unsafe.Add(ref d0, 2) = rgb3; - Unsafe.Add(ref d0, 3) = rgb4; - } - - int slice = (int)count * Vector256.Count; - redChannel = redChannel[slice..]; - greenChannel = greenChannel[slice..]; - blueChannel = blueChannel[slice..]; - destination = destination[slice..]; - } - - internal static void UnpackToRgbPlanesAvx2Reduce( - ref Span redChannel, - ref Span greenChannel, - ref Span blueChannel, - ref ReadOnlySpan source) - { - ref Vector256 rgbByteSpan = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); - ref Vector256 destRRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(redChannel)); - ref Vector256 destGRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(greenChannel)); - ref Vector256 destBRef = ref Unsafe.As>(ref MemoryMarshal.GetReference(blueChannel)); - - Vector256 extractToLanesMask = MoveFirst24BytesToSeparateLanes(); - Vector256 extractRgbMask = ExtractRgb(); - Vector256 rgb, rg, bx; - Vector256 r, g, b; - - // Each iteration consumes 8 Rgb24 pixels (24 bytes) but starts with a 32-byte load, - // so we need 3 extra pixels of addressable slack beyond the vectorized chunk. - const int bytesPerRgbStride = 24; - nuint count = source.Length > 3 ? (uint)(source.Length - 3) / 8 : 0; - for (nuint i = 0; i < count; i++) - { - rgb = Avx2.PermuteVar8x32(Unsafe.AddByteOffset(ref rgbByteSpan, (uint)(bytesPerRgbStride * i)).AsUInt32(), extractToLanesMask).AsByte(); - - rgb = Avx2.Shuffle(rgb, extractRgbMask); - - rg = Avx2.UnpackLow(rgb, Vector256.Zero); - bx = Avx2.UnpackHigh(rgb, Vector256.Zero); - - r = Avx.ConvertToVector256Single(Avx2.UnpackLow(rg, Vector256.Zero).AsInt32()); - g = Avx.ConvertToVector256Single(Avx2.UnpackHigh(rg, Vector256.Zero).AsInt32()); - b = Avx.ConvertToVector256Single(Avx2.UnpackLow(bx, Vector256.Zero).AsInt32()); - - Unsafe.Add(ref destRRef, i) = r; - Unsafe.Add(ref destGRef, i) = g; - Unsafe.Add(ref destBRef, i) = b; - } - - int sliceCount = (int)(count * 8); - redChannel = redChannel[sliceCount..]; - greenChannel = greenChannel[sliceCount..]; - blueChannel = blueChannel[sliceCount..]; - source = source[sliceCount..]; - } } } diff --git a/src/ImageSharp/Common/Helpers/SimdUtils.Pack.cs b/src/ImageSharp/Common/Helpers/SimdUtils.Pack.cs index f471d0231..71afd3a74 100644 --- a/src/ImageSharp/Common/Helpers/SimdUtils.Pack.cs +++ b/src/ImageSharp/Common/Helpers/SimdUtils.Pack.cs @@ -3,7 +3,9 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp; @@ -19,11 +21,16 @@ internal static partial class SimdUtils { DebugGuard.IsTrue(greenChannel.Length == redChannel.Length, nameof(greenChannel), "Channels must be of same size!"); DebugGuard.IsTrue(blueChannel.Length == redChannel.Length, nameof(blueChannel), "Channels must be of same size!"); - DebugGuard.IsTrue(destination.Length > redChannel.Length + 2, nameof(destination), "'destination' must contain a padding of 3 elements!"); + DebugGuard.IsTrue(destination.Length >= redChannel.Length, nameof(destination), "'destination' span should not be shorter than the source channels!"); if (Avx2.IsSupported) { - HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination); + HwIntrinsics.PackFromRgbPlanesReduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination); + } + + if (Vector128.IsHardwareAccelerated) + { + PackFromRgbPlanesVector128Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination); } else { @@ -42,11 +49,16 @@ internal static partial class SimdUtils { DebugGuard.IsTrue(greenChannel.Length == redChannel.Length, nameof(greenChannel), "Channels must be of same size!"); DebugGuard.IsTrue(blueChannel.Length == redChannel.Length, nameof(blueChannel), "Channels must be of same size!"); - DebugGuard.IsTrue(destination.Length > redChannel.Length, nameof(destination), "'destination' span should not be shorter than the source channels!"); + DebugGuard.IsTrue(destination.Length >= redChannel.Length, nameof(destination), "'destination' span should not be shorter than the source channels!"); if (Avx2.IsSupported) { - HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination); + HwIntrinsics.PackFromRgbPlanesReduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination); + } + + if (Vector128.IsHardwareAccelerated) + { + PackFromRgbPlanesVector128Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref destination); } else { @@ -69,12 +81,143 @@ internal static partial class SimdUtils if (Avx2.IsSupported) { - HwIntrinsics.UnpackToRgbPlanesAvx2Reduce(ref redChannel, ref greenChannel, ref blueChannel, ref source); + HwIntrinsics.UnpackToRgbPlanesReduce(ref redChannel, ref greenChannel, ref blueChannel, ref source); } UnpackToRgbPlanesScalar(redChannel, greenChannel, blueChannel, source); } + /// + /// Packs complete sixteen-pixel batches into exact-length storage using portable 128-bit SIMD. + /// + /// The red source span, advanced past the converted batches. + /// The green source span, advanced past the converted batches. + /// The blue source span, advanced past the converted batches. + /// The destination span, advanced past the converted batches. + private static void PackFromRgbPlanesVector128Reduce( + ref ReadOnlySpan redChannel, + ref ReadOnlySpan greenChannel, + ref ReadOnlySpan blueChannel, + ref Span destination) + { + ref byte redBase = ref MemoryMarshal.GetReference(redChannel); + ref byte greenBase = ref MemoryMarshal.GetReference(greenChannel); + ref byte blueBase = ref MemoryMarshal.GetReference(blueChannel); + ref byte destinationBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + Vector128 opaqueAlpha = Vector128.Create(byte.MaxValue); + Vector128 removeAlpha = Vector128.Create((byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, byte.MaxValue, byte.MaxValue, byte.MaxValue, byte.MaxValue); + nuint batchCount = (nuint)(uint)redChannel.Length / (uint)Vector128.Count; + + for (nuint i = 0; i < batchCount; i++) + { + nuint sourceOffset = i * (uint)Vector128.Count; + Vector128 red = Vector128.LoadUnsafe(ref redBase, sourceOffset); + Vector128 green = Vector128.LoadUnsafe(ref greenBase, sourceOffset); + Vector128 blue = Vector128.LoadUnsafe(ref blueBase, sourceOffset); + InterleaveRgbPlanes(red, green, blue, opaqueAlpha, out Vector128 rgba0, out Vector128 rgba1, out Vector128 rgba2, out Vector128 rgba3); + + // The native byte shuffle removes alpha from four pixels at a time. Each result owns twelve bytes, so + // exact stores avoid coupling the SIMD path to padding beyond the row or the next memory-group segment. + ref byte destination0 = ref Unsafe.Add(ref destinationBase, i * 48); + StoreRgb24Batch(Vector128.ShuffleNative(rgba0, removeAlpha), ref destination0); + StoreRgb24Batch(Vector128.ShuffleNative(rgba1, removeAlpha), ref Unsafe.Add(ref destination0, 12)); + StoreRgb24Batch(Vector128.ShuffleNative(rgba2, removeAlpha), ref Unsafe.Add(ref destination0, 24)); + StoreRgb24Batch(Vector128.ShuffleNative(rgba3, removeAlpha), ref Unsafe.Add(ref destination0, 36)); + } + + int convertedCount = (int)(batchCount * (uint)Vector128.Count); + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + destination = destination[convertedCount..]; + } + + /// + /// Packs complete sixteen-pixel batches into exact-length storage using portable 128-bit SIMD. + /// + /// The red source span, advanced past the converted batches. + /// The green source span, advanced past the converted batches. + /// The blue source span, advanced past the converted batches. + /// The destination span, advanced past the converted batches. + private static void PackFromRgbPlanesVector128Reduce( + ref ReadOnlySpan redChannel, + ref ReadOnlySpan greenChannel, + ref ReadOnlySpan blueChannel, + ref Span destination) + { + ref byte redBase = ref MemoryMarshal.GetReference(redChannel); + ref byte greenBase = ref MemoryMarshal.GetReference(greenChannel); + ref byte blueBase = ref MemoryMarshal.GetReference(blueChannel); + ref Vector128 destinationBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); + Vector128 opaqueAlpha = Vector128.Create(byte.MaxValue); + nuint batchCount = (nuint)(uint)redChannel.Length / (uint)Vector128.Count; + + for (nuint i = 0; i < batchCount; i++) + { + nuint sourceOffset = i * (uint)Vector128.Count; + Vector128 red = Vector128.LoadUnsafe(ref redBase, sourceOffset); + Vector128 green = Vector128.LoadUnsafe(ref greenBase, sourceOffset); + Vector128 blue = Vector128.LoadUnsafe(ref blueBase, sourceOffset); + InterleaveRgbPlanes(red, green, blue, opaqueAlpha, out Vector128 rgba0, out Vector128 rgba1, out Vector128 rgba2, out Vector128 rgba3); + + ref Vector128 destination0 = ref Unsafe.Add(ref destinationBase, i * 4); + destination0 = rgba0; + Unsafe.Add(ref destination0, 1) = rgba1; + Unsafe.Add(ref destination0, 2) = rgba2; + Unsafe.Add(ref destination0, 3) = rgba3; + } + + int convertedCount = (int)(batchCount * (uint)Vector128.Count); + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + destination = destination[convertedCount..]; + } + + /// + /// Interleaves sixteen planar RGB samples into four groups of four opaque RGBA pixels. + /// + /// The red component lanes. + /// The green component lanes. + /// The blue component lanes. + /// The opaque alpha lanes. + /// The first four interleaved pixels. + /// The second four interleaved pixels. + /// The third four interleaved pixels. + /// The fourth four interleaved pixels. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InterleaveRgbPlanes( + Vector128 red, + Vector128 green, + Vector128 blue, + Vector128 alpha, + out Vector128 rgba0, + out Vector128 rgba1, + out Vector128 rgba2, + out Vector128 rgba3) + { + Vector128 redGreenLow = Vector128_.UnpackLow(red, green); + Vector128 redGreenHigh = Vector128_.UnpackHigh(red, green); + Vector128 blueAlphaLow = Vector128_.UnpackLow(blue, alpha); + Vector128 blueAlphaHigh = Vector128_.UnpackHigh(blue, alpha); + rgba0 = Vector128_.UnpackLow(redGreenLow.AsInt16(), blueAlphaLow.AsInt16()).AsByte(); + rgba1 = Vector128_.UnpackHigh(redGreenLow.AsInt16(), blueAlphaLow.AsInt16()).AsByte(); + rgba2 = Vector128_.UnpackLow(redGreenHigh.AsInt16(), blueAlphaHigh.AsInt16()).AsByte(); + rgba3 = Vector128_.UnpackHigh(redGreenHigh.AsInt16(), blueAlphaHigh.AsInt16()).AsByte(); + } + + /// + /// Stores the twelve packed RGB bytes in one shuffled SIMD value without writing its unused lanes. + /// + /// The packed RGB bytes in the first twelve lanes. + /// The first destination byte. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreRgb24Batch(Vector128 value, ref byte destination) + { + Unsafe.WriteUnaligned(ref destination, value.AsUInt64().ToScalar()); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 8), value.AsUInt32().GetElement(2)); + } + private static void PackFromRgbPlanesScalarBatchedReduce( ref ReadOnlySpan redChannel, ref ReadOnlySpan greenChannel, @@ -86,8 +229,8 @@ internal static partial class SimdUtils ref ByteTuple4 b = ref Unsafe.As(ref MemoryMarshal.GetReference(blueChannel)); ref Rgb24 rgb = ref MemoryMarshal.GetReference(destination); - nuint count = (uint)redChannel.Length / 4; - for (nuint i = 0; i < count; i++) + nuint batchCount = (uint)redChannel.Length / 4; + for (nuint i = 0; i < batchCount; i++) { ref Rgb24 d0 = ref Unsafe.Add(ref rgb, i * 4); ref Rgb24 d1 = ref Unsafe.Add(ref d0, 1); @@ -115,11 +258,11 @@ internal static partial class SimdUtils d3.B = bb.V3; } - int finished = (int)(count * 4); - redChannel = redChannel[finished..]; - greenChannel = greenChannel[finished..]; - blueChannel = blueChannel[finished..]; - destination = destination[finished..]; + int convertedCount = (int)(batchCount * 4); + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + destination = destination[convertedCount..]; } private static void PackFromRgbPlanesScalarBatchedReduce( @@ -133,9 +276,9 @@ internal static partial class SimdUtils ref ByteTuple4 b = ref Unsafe.As(ref MemoryMarshal.GetReference(blueChannel)); ref Rgba32 rgb = ref MemoryMarshal.GetReference(destination); - nuint count = (uint)redChannel.Length / 4; + nuint batchCount = (uint)redChannel.Length / 4; destination.Fill(new Rgba32(0, 0, 0, 255)); - for (nuint i = 0; i < count; i++) + for (nuint i = 0; i < batchCount; i++) { ref Rgba32 d0 = ref Unsafe.Add(ref rgb, i * 4); ref Rgba32 d1 = ref Unsafe.Add(ref d0, 1); @@ -163,11 +306,11 @@ internal static partial class SimdUtils d3.B = bb.V3; } - int finished = (int)(count * 4); - redChannel = redChannel[finished..]; - greenChannel = greenChannel[finished..]; - blueChannel = blueChannel[finished..]; - destination = destination[finished..]; + int convertedCount = (int)(batchCount * 4); + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + destination = destination[convertedCount..]; } private static void PackFromRgbPlanesRemainder( @@ -181,7 +324,7 @@ internal static partial class SimdUtils ref byte b = ref MemoryMarshal.GetReference(blueChannel); ref Rgb24 rgb = ref MemoryMarshal.GetReference(destination); - for (nuint i = 0; i < (uint)destination.Length; i++) + for (nuint i = 0; i < (uint)redChannel.Length; i++) { ref Rgb24 d = ref Unsafe.Add(ref rgb, i); d.R = Unsafe.Add(ref r, i); @@ -201,7 +344,7 @@ internal static partial class SimdUtils ref byte b = ref MemoryMarshal.GetReference(blueChannel); ref Rgba32 rgba = ref MemoryMarshal.GetReference(destination); - for (nuint i = 0; i < (uint)destination.Length; i++) + for (nuint i = 0; i < (uint)redChannel.Length; i++) { ref Rgba32 d = ref Unsafe.Add(ref rgba, i); d.R = Unsafe.Add(ref r, i); @@ -234,4 +377,291 @@ internal static partial class SimdUtils Unsafe.Add(ref b, i) = src.B; } } + + /// + /// Provides the hardware-intrinsic reducers used by the planar RGB packing pipeline. + /// + public static partial class HwIntrinsics + { + /// + /// Creates the AVX2 lane order used before interleaving planar RGB components. + /// + /// The source lane permutation. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 PermuteMaskEvenOdd8x32() => Vector256.Create(0u, 2, 4, 6, 1, 3, 5, 7); + + /// + /// Packs complete AVX2 batches into pixels and retains the unconverted remainder. + /// + /// The red source span. + /// The green source span. + /// The blue source span. + /// The destination pixel span. + internal static void PackFromRgbPlanesReduce( + ref ReadOnlySpan redChannel, + ref ReadOnlySpan greenChannel, + ref ReadOnlySpan blueChannel, + ref Span destination) + { + ref Vector256 redBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(redChannel)); + ref Vector256 greenBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(greenChannel)); + ref Vector256 blueBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(blueChannel)); + ref byte destinationBase = ref Unsafe.As(ref MemoryMarshal.GetReference(destination)); + nuint batchCount = redChannel.Vector256Count(); + Vector256 sourceOrder = PermuteMaskEvenOdd8x32(); + Vector256 packedOrder = Vector256.Create(0u, 1, 2, 4, 5, 6, 3, 7); + Vector256 opaqueAlpha = Vector256.Create(byte.MaxValue); + Vector128 removeAlphaLower = Vector128.Create((byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15); + Vector128 removeAlphaUpper = Vector128.Create((byte)16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30, 19, 23, 27, 31); + Vector256 removeAlpha = Vector256.Create(removeAlphaLower, removeAlphaUpper); + + bool hasWritablePadding = destination.Length >= redChannel.Length + 3; + nuint i = 0; + + // Non-final batches retain the original four overlapping wide stores. Splitting the final batch keeps + // the exact-row decision out of the hot loop and limits the narrower stores to the only bytes that can + // cross the destination boundary. + for (; i + 1 < batchCount; i++) + { + PackRgb24Batch( + Unsafe.Add(ref redBase, i), + Unsafe.Add(ref greenBase, i), + Unsafe.Add(ref blueBase, i), + opaqueAlpha, + sourceOrder, + packedOrder, + removeAlpha, + out Vector256 rgb0, + out Vector256 rgb1, + out Vector256 rgb2, + out Vector256 rgb3); + + ref byte destination0 = ref Unsafe.Add(ref destinationBase, 96 * i); + ref byte destination1 = ref Unsafe.Add(ref destination0, 24); + ref byte destination2 = ref Unsafe.Add(ref destination1, 24); + ref byte destination3 = ref Unsafe.Add(ref destination2, 24); + + Unsafe.As>(ref destination0) = rgb0; + Unsafe.As>(ref destination1) = rgb1; + Unsafe.As>(ref destination2) = rgb2; + Unsafe.As>(ref destination3) = rgb3; + } + + if (i < batchCount) + { + PackRgb24Batch( + Unsafe.Add(ref redBase, i), + Unsafe.Add(ref greenBase, i), + Unsafe.Add(ref blueBase, i), + opaqueAlpha, + sourceOrder, + packedOrder, + removeAlpha, + out Vector256 rgb0, + out Vector256 rgb1, + out Vector256 rgb2, + out Vector256 rgb3); + + ref byte destination0 = ref Unsafe.Add(ref destinationBase, 96 * i); + ref byte destination1 = ref Unsafe.Add(ref destination0, 24); + ref byte destination2 = ref Unsafe.Add(ref destination1, 24); + ref byte destination3 = ref Unsafe.Add(ref destination2, 24); + + Unsafe.As>(ref destination0) = rgb0; + Unsafe.As>(ref destination1) = rgb1; + Unsafe.As>(ref destination2) = rgb2; + + if (hasWritablePadding) + { + Unsafe.As>(ref destination3) = rgb3; + } + else + { + // The final compacted vector contains 24 RGB bytes followed by eight unused bytes. Exact stores + // retain all useful bytes without writing beyond an unpadded destination row. + Unsafe.As>(ref destination3) = rgb3.GetLower(); + Unsafe.As(ref Unsafe.Add(ref destination3, 16)) = rgb3.GetUpper().AsUInt64().ToScalar(); + } + } + + int convertedCount = (int)batchCount * Vector256.Count; + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + destination = destination[convertedCount..]; + } + + /// + /// Packs complete AVX2 batches into pixels and retains the unconverted remainder. + /// + /// The red source span. + /// The green source span. + /// The blue source span. + /// The destination pixel span. + internal static void PackFromRgbPlanesReduce( + ref ReadOnlySpan redChannel, + ref ReadOnlySpan greenChannel, + ref ReadOnlySpan blueChannel, + ref Span destination) + { + ref Vector256 redBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(redChannel)); + ref Vector256 greenBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(greenChannel)); + ref Vector256 blueBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(blueChannel)); + ref Vector256 destinationBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(destination)); + nuint batchCount = redChannel.Vector256Count(); + Vector256 sourceOrder = PermuteMaskEvenOdd8x32(); + Vector256 opaqueAlpha = Vector256.Create(byte.MaxValue); + + for (nuint i = 0; i < batchCount; i++) + { + InterleaveRgbPlanes( + Unsafe.Add(ref redBase, i), + Unsafe.Add(ref greenBase, i), + Unsafe.Add(ref blueBase, i), + opaqueAlpha, + sourceOrder, + out Vector256 rgba0, + out Vector256 rgba1, + out Vector256 rgba2, + out Vector256 rgba3); + + ref Vector256 destination0 = ref Unsafe.Add(ref destinationBase, i * 4); + destination0 = rgba0; + Unsafe.Add(ref destination0, 1) = rgba1; + Unsafe.Add(ref destination0, 2) = rgba2; + Unsafe.Add(ref destination0, 3) = rgba3; + } + + int convertedCount = (int)batchCount * Vector256.Count; + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + destination = destination[convertedCount..]; + } + + /// + /// Unpacks complete AVX2 batches from pixels and retains the unconverted remainder. + /// + /// The red destination span. + /// The green destination span. + /// The blue destination span. + /// The source pixel span. + internal static void UnpackToRgbPlanesReduce( + ref Span redChannel, + ref Span greenChannel, + ref Span blueChannel, + ref ReadOnlySpan source) + { + ref Vector256 sourceBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(source)); + ref Vector256 redBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(redChannel)); + ref Vector256 greenBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(greenChannel)); + ref Vector256 blueBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(blueChannel)); + Vector256 separateLanes = Vector256.Create(0u, 1, 2, 6, 3, 4, 5, 7); + Vector128 extractRgbLower = Vector128.Create((byte)0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, byte.MaxValue, byte.MaxValue, byte.MaxValue, byte.MaxValue); + Vector128 extractRgbUpper = Vector128.Create((byte)16, 19, 22, 25, 17, 20, 23, 26, 18, 21, 24, 27, byte.MaxValue, byte.MaxValue, byte.MaxValue, byte.MaxValue); + Vector256 extractRgb = Vector256.Create(extractRgbLower, extractRgbUpper); + + // Each iteration consumes eight Rgb24 pixels, or 24 bytes, but starts with a 32-byte load. Three extra + // source pixels must therefore remain addressable beyond every vectorized batch. + const int bytesPerBatch = 24; + nuint batchCount = source.Length > 3 ? (uint)(source.Length - 3) / 8 : 0; + + for (nuint i = 0; i < batchCount; i++) + { + Vector256 packed = Avx2.PermuteVar8x32(Unsafe.AddByteOffset(ref sourceBase, (uint)(bytesPerBatch * i)).AsUInt32(), separateLanes).AsByte(); + packed = Vector256.ShuffleNative(packed, extractRgb); + + Vector256 redGreen = Avx2.UnpackLow(packed, Vector256.Zero); + Vector256 blue = Avx2.UnpackHigh(packed, Vector256.Zero); + Vector256 red = Avx.ConvertToVector256Single(Avx2.UnpackLow(redGreen, Vector256.Zero).AsInt32()); + Vector256 green = Avx.ConvertToVector256Single(Avx2.UnpackHigh(redGreen, Vector256.Zero).AsInt32()); + Vector256 blueValues = Avx.ConvertToVector256Single(Avx2.UnpackLow(blue, Vector256.Zero).AsInt32()); + + Unsafe.Add(ref redBase, i) = red; + Unsafe.Add(ref greenBase, i) = green; + Unsafe.Add(ref blueBase, i) = blueValues; + } + + int convertedCount = (int)(batchCount * 8); + redChannel = redChannel[convertedCount..]; + greenChannel = greenChannel[convertedCount..]; + blueChannel = blueChannel[convertedCount..]; + source = source[convertedCount..]; + } + + /// + /// Interleaves and compacts one AVX2 batch into four groups of eight pixels. + /// + /// The red component lanes. + /// The green component lanes. + /// The blue component lanes. + /// The opaque alpha lanes used during interleaving. + /// The cross-lane source permutation. + /// The cross-lane packed RGB permutation. + /// The native byte-shuffle indices that compact RGBA to RGB. + /// The first eight packed pixels. + /// The second eight packed pixels. + /// The third eight packed pixels. + /// The fourth eight packed pixels. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void PackRgb24Batch( + Vector256 red, + Vector256 green, + Vector256 blue, + Vector256 alpha, + Vector256 sourceOrder, + Vector256 packedOrder, + Vector256 removeAlpha, + out Vector256 rgb0, + out Vector256 rgb1, + out Vector256 rgb2, + out Vector256 rgb3) + { + InterleaveRgbPlanes(red, green, blue, alpha, sourceOrder, out Vector256 rgba0, out Vector256 rgba1, out Vector256 rgba2, out Vector256 rgba3); + + rgb0 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba0, removeAlpha).AsUInt32(), packedOrder).AsByte(); + rgb1 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba1, removeAlpha).AsUInt32(), packedOrder).AsByte(); + rgb2 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba2, removeAlpha).AsUInt32(), packedOrder).AsByte(); + rgb3 = Avx2.PermuteVar8x32(Vector256.ShuffleNative(rgba3, removeAlpha).AsUInt32(), packedOrder).AsByte(); + } + + /// + /// Interleaves 32 planar RGB samples into four groups of eight opaque RGBA pixels. + /// + /// The red component lanes. + /// The green component lanes. + /// The blue component lanes. + /// The opaque alpha lanes. + /// The cross-lane source permutation. + /// The first eight interleaved pixels. + /// The second eight interleaved pixels. + /// The third eight interleaved pixels. + /// The fourth eight interleaved pixels. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InterleaveRgbPlanes( + Vector256 red, + Vector256 green, + Vector256 blue, + Vector256 alpha, + Vector256 sourceOrder, + out Vector256 rgba0, + out Vector256 rgba1, + out Vector256 rgba2, + out Vector256 rgba3) + { + red = Avx2.PermuteVar8x32(red.AsUInt32(), sourceOrder).AsByte(); + green = Avx2.PermuteVar8x32(green.AsUInt32(), sourceOrder).AsByte(); + blue = Avx2.PermuteVar8x32(blue.AsUInt32(), sourceOrder).AsByte(); + + Vector256 redGreenLow = Avx2.UnpackLow(red, green); + Vector256 redGreenHigh = Avx2.UnpackHigh(red, green); + Vector256 blueAlphaLow = Avx2.UnpackLow(blue, alpha); + Vector256 blueAlphaHigh = Avx2.UnpackHigh(blue, alpha); + + rgba0 = Avx2.UnpackLow(redGreenLow.AsUInt16(), blueAlphaLow.AsUInt16()).AsByte(); + rgba1 = Avx2.UnpackHigh(redGreenLow.AsUInt16(), blueAlphaLow.AsUInt16()).AsByte(); + rgba2 = Avx2.UnpackLow(redGreenHigh.AsUInt16(), blueAlphaHigh.AsUInt16()).AsByte(); + rgba3 = Avx2.UnpackHigh(redGreenHigh.AsUInt16(), blueAlphaHigh.AsUInt16()).AsByte(); + } + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.RowConverters.cs b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.RowConverters.cs index c2e38c499..25ba6f833 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.RowConverters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.RowConverters.cs @@ -129,8 +129,7 @@ internal static partial class Av1YuvConverter /// /// The row index. /// The reusable conversion buffer. - /// The padded destination used when an eight-bit image row cannot expose sufficient padding. - public void Convert(int y, Span span, Span proxy) + public void Convert(int y, Span span) { int width = this.image.Width; Span red = span[..width]; @@ -203,15 +202,7 @@ internal static partial class Av1YuvConverter SimdUtils.NormalizedFloatToByteSaturate(green, greenBytes); SimdUtils.NormalizedFloatToByteSaturate(blue, blueBytes); - if (this.image.PixelBuffer.DangerousTryGetPaddedRowSpan(y, 3, out Span paddedDestination)) - { - PixelOperations.Instance.PackFromRgbPlanes(redBytes, greenBytes, blueBytes, paddedDestination); - } - else - { - PixelOperations.Instance.PackFromRgbPlanes(redBytes, greenBytes, blueBytes, proxy); - proxy[..width].CopyTo(destination); - } + PixelOperations.Instance.PackFromRgbPlanes(redBytes, greenBytes, blueBytes, destination); } else { diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs index 616d25d2a..7aff4f315 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs @@ -44,12 +44,10 @@ internal static partial class Av1YuvConverter { YuvToRgbRowConverter converter = new(configuration, frameBuffer, image, colorConverter); using IMemoryOwner scratchOwner = configuration.MemoryAllocator.Allocate(converter.BufferLength); - using IMemoryOwner proxyOwner = configuration.MemoryAllocator.Allocate(image.Width + 3); Span scratch = scratchOwner.GetSpan(); - Span proxy = proxyOwner.GetSpan()[..(image.Width + 3)]; for (int y = 0; y < image.Height; y++) { - converter.Convert(y, scratch, proxy); + converter.Convert(y, scratch); } } else @@ -59,7 +57,7 @@ internal static partial class Av1YuvConverter Span scratch = owner.GetSpan(); for (int y = 0; y < image.Height; y++) { - converter.Convert(y, scratch, Span.Empty); + converter.Convert(y, scratch); } } } diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs index 2bd4b95fd..d70e85476 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs @@ -42,11 +42,6 @@ internal class SpectralConverter : SpectralConverter, IDisposable /// private IMemoryOwner rgbBuffer; - /// - /// Proxy buffer used in packing from RGB to target TPixel pixels. - /// - private IMemoryOwner paddedProxyPixelRow; - /// /// Resulting 2D pixel buffer. /// @@ -163,19 +158,8 @@ internal class SpectralConverter : SpectralConverter, IDisposable SimdUtils.NormalizedFloatToByteSaturate(values.Component1, g); SimdUtils.NormalizedFloatToByteSaturate(values.Component2, b); - // PackFromRgbPlanes expects the destination to be padded, so try to get padded span containing extra elements from the next row. - // If we can't get such a padded row because we are on a MemoryGroup boundary or at the last row, - // pack pixels to a temporary, padded proxy buffer, then copy the relevant values to the destination row. - if (this.pixelBuffer.DangerousTryGetPaddedRowSpan(yy, 3, out Span destRow)) - { - PixelOperations.Instance.PackFromRgbPlanes(r, g, b, destRow); - } - else - { - Span proxyRow = this.paddedProxyPixelRow.GetSpan(); - PixelOperations.Instance.PackFromRgbPlanes(r, g, b, proxyRow); - proxyRow[..width].CopyTo(this.pixelBuffer.DangerousGetRowSpan(yy)); - } + Span destination = this.pixelBuffer.DangerousGetRowSpan(yy); + PixelOperations.Instance.PackFromRgbPlanes(r, g, b, destination); } this.pixelRowCounter += this.pixelRowsPerStep; @@ -214,7 +198,6 @@ internal class SpectralConverter : SpectralConverter, IDisposable pixelSize.Height, this.Configuration.PreferContiguousImageBuffers, AllocationOptions.Clean); - this.paddedProxyPixelRow = allocator.Allocate(pixelSize.Width + 3); // Component processors from spectral to RGB int bufferWidth = majorBlockWidth * blockPixelSize; @@ -275,7 +258,6 @@ internal class SpectralConverter : SpectralConverter, IDisposable } this.rgbBuffer?.Dispose(); - this.paddedProxyPixelRow?.Dispose(); this.pixelBuffer?.Dispose(); } } diff --git a/src/ImageSharp/PixelFormats/PixelOperations{TPixel}.cs b/src/ImageSharp/PixelFormats/PixelOperations{TPixel}.cs index 63ece4188..e5faf976c 100644 --- a/src/ImageSharp/PixelFormats/PixelOperations{TPixel}.cs +++ b/src/ImageSharp/PixelFormats/PixelOperations{TPixel}.cs @@ -348,7 +348,6 @@ public partial class PixelOperations /// /// Bulk operation that packs 3 separate RGB channels to . - /// The destination must have a padding of 3. /// /// A to the red values. /// A to the green values. @@ -424,6 +423,6 @@ public partial class PixelOperations { Guard.IsTrue(greenChannel.Length == count, nameof(greenChannel), "Channels must be of same size!"); Guard.IsTrue(blueChannel.Length == count, nameof(blueChannel), "Channels must be of same size!"); - Guard.IsTrue(destination.Length > count + 2, nameof(destination), "'destination' must contain a padding of 3 elements!"); + Guard.IsTrue(destination.Length >= count, nameof(destination), "'destination' span should not be shorter than the source channels!"); } } diff --git a/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs b/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs index 226dcc777..636a9c4a1 100644 --- a/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs +++ b/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs @@ -244,7 +244,7 @@ public unsafe class PixelConversion_PackFromRgbPlanes ReadOnlySpan g = this.rBuf; ReadOnlySpan b = this.rBuf; Span rgb = this.rgbBuf; - SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref r, ref g, ref b, ref rgb); + SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref r, ref g, ref b, ref rgb); } [Benchmark] @@ -254,7 +254,7 @@ public unsafe class PixelConversion_PackFromRgbPlanes ReadOnlySpan g = this.rBuf; ReadOnlySpan b = this.rBuf; Span rgb = this.rgbaBuf; - SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref r, ref g, ref b, ref rgb); + SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref r, ref g, ref b, ref rgb); } #pragma warning disable SA1132 diff --git a/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanesExact.cs b/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanesExact.cs new file mode 100644 index 000000000..45f476493 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanesExact.cs @@ -0,0 +1,64 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.PixelFormats; + +namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion; + +/// +/// Measures planar RGB packing for padded and exact-length decoder rows. +/// +public class PixelConversion_PackFromRgbPlanesExact +{ + private byte[] red; + private byte[] green; + private byte[] blue; + private Rgb24[] exactDestination; + private Rgb24[] paddedDestination; + + /// + /// Gets or sets the decoded row width. + /// + [Params(1920, 4242)] + public int Count { get; set; } + + /// + /// Creates deterministic component planes and destination rows outside the measured operation. + /// + [GlobalSetup] + public void Setup() + { + this.red = new byte[this.Count]; + this.green = new byte[this.Count]; + this.blue = new byte[this.Count]; + this.exactDestination = new Rgb24[this.Count]; + this.paddedDestination = new Rgb24[this.Count + 3]; + + new Random(42).NextBytes(this.red); + new Random(43).NextBytes(this.green); + new Random(44).NextBytes(this.blue); + } + + /// + /// Packs a row using the legacy decoder contract with three writable destination pixels beyond the row. + /// + [Benchmark(Baseline = true)] + public void PaddedDestination() => SimdUtils.PackFromRgbPlanes(this.red, this.green, this.blue, this.paddedDestination); + + /// + /// Packs through a padded proxy and copies the completed row, matching the former decoder fallback. + /// + [Benchmark] + public void PaddedProxyAndCopy() + { + SimdUtils.PackFromRgbPlanes(this.red, this.green, this.blue, this.paddedDestination); + this.paddedDestination.AsSpan(0, this.Count).CopyTo(this.exactDestination); + } + + /// + /// Packs a row directly into the exact-length destination exposed by an image frame. + /// + [Benchmark] + public void ExactDestination() => SimdUtils.PackFromRgbPlanes(this.red, this.green, this.blue, this.exactDestination); +} diff --git a/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs b/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs index b4fbf5fd8..7b7b1b371 100644 --- a/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs +++ b/tests/ImageSharp.Tests/Common/SimdUtilsTests.cs @@ -272,7 +272,7 @@ public partial class SimdUtilsTests SimdUtils.PackFromRgbPlanes(r, g, b, actual)); [Fact] - public void PackFromRgbPlanesAvx2Reduce_Rgb24() + public void PackFromRgbPlanesReduce_Rgb24() { if (!Avx2.IsSupported) { @@ -282,15 +282,14 @@ public partial class SimdUtilsTests byte[] r = [.. Enumerable.Range(0, 32).Select(x => (byte)x)]; byte[] g = [.. Enumerable.Range(100, 32).Select(x => (byte)x)]; byte[] b = [.. Enumerable.Range(200, 32).Select(x => (byte)x)]; - const int padding = 4; - Rgb24[] d = new Rgb24[32 + padding]; + Rgb24[] d = new Rgb24[32]; ReadOnlySpan rr = r.AsSpan(); ReadOnlySpan gg = g.AsSpan(); ReadOnlySpan bb = b.AsSpan(); Span dd = d.AsSpan(); - SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref rr, ref gg, ref bb, ref dd); + SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref rr, ref gg, ref bb, ref dd); for (int i = 0; i < 32; i++) { @@ -302,11 +301,11 @@ public partial class SimdUtilsTests Assert.Equal(0, rr.Length); Assert.Equal(0, gg.Length); Assert.Equal(0, bb.Length); - Assert.Equal(padding, dd.Length); + Assert.Equal(0, dd.Length); } [Fact] - public void PackFromRgbPlanesAvx2Reduce_Rgba32() + public void PackFromRgbPlanesReduce_Rgba32() { if (!Avx2.IsSupported) { @@ -324,7 +323,7 @@ public partial class SimdUtilsTests ReadOnlySpan bb = b.AsSpan(); Span dd = d.AsSpan(); - SimdUtils.HwIntrinsics.PackFromRgbPlanesAvx2Reduce(ref rr, ref gg, ref bb, ref dd); + SimdUtils.HwIntrinsics.PackFromRgbPlanesReduce(ref rr, ref gg, ref bb, ref dd); for (int i = 0; i < 32; i++) { @@ -354,10 +353,10 @@ public partial class SimdUtilsTests expected[i] = TPixel.FromRgb24(new Rgb24(r[i], g[i], b[i])); } - TPixel[] actual = new TPixel[count + 3]; // padding for Rgb24 AVX2 + TPixel[] actual = new TPixel[count]; packMethod(r, g, b, actual); - Assert.True(expected.AsSpan().SequenceEqual(actual.AsSpan()[..count])); + Assert.True(expected.AsSpan().SequenceEqual(actual)); } private static void TestImpl_BulkConvertNormalizedFloatToByteClampOverflows(