Browse Source

Clean up normalized SIMD pipelines

pull/3161/head
James Jackson-South 3 weeks ago
parent
commit
1c726da0df
  1. 9
      src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs
  2. 33
      src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
  3. 15
      src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
  4. 219
      src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs
  5. 39
      src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
  6. 94
      src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs
  7. 19
      src/ImageSharp/Common/Helpers/Vector128Utilities.cs
  8. 6
      src/ImageSharp/Common/Helpers/Vector256Utilities.cs
  9. 20
      src/ImageSharp/Common/Helpers/Vector512Utilities.cs
  10. 90
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.CmykOperator.cs
  11. 105
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.GrayScaleOperator.cs
  12. 167
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Operator.cs
  13. 305
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Packing.cs
  14. 90
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.RgbOperator.cs
  15. 150
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YCbCrOperator.cs
  16. 4
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YccKOperator.cs
  17. 135
      src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverterBase.cs
  18. 4
      src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs
  19. 7
      src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
  20. 341
      src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs
  21. 7
      src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
  22. 121
      src/ImageSharp/Formats/Png/Filters/PngFilterEncoder.cs
  23. 12
      src/ImageSharp/Formats/Png/Filters/SubFilter.cs
  24. 10
      src/ImageSharp/Formats/Png/Filters/UpFilter.cs
  25. 1
      src/ImageSharp/Formats/Webp/AlphaDecoder.cs
  26. 18
      src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs
  27. 1080
      src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPixelBlenders.Generated.cs
  28. 10
      src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPixelBlenders.Generated.tt
  29. 5
      src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPixelBlender{TPixel,TOperator}.cs
  30. 1080
      src/ImageSharp/PixelFormats/PixelBlenders/DefaultPixelBlenders.Generated.cs
  31. 10
      src/ImageSharp/PixelFormats/PixelBlenders/DefaultPixelBlenders.Generated.tt
  32. 10
      src/ImageSharp/PixelFormats/PixelBlenders/IPixelBlenderOperator.cs
  33. 302
      src/ImageSharp/PixelFormats/PixelBlenders/PixelBlender{TPixel,TOperator}.cs
  34. 9
      src/ImageSharp/PixelFormats/Utils/Vector4Converters.Affine.cs
  35. 8
      src/ImageSharp/PixelFormats/Utils/Vector4Converters.AffineOperators.cs
  36. 3
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/CmykColorConversion.cs
  37. 3
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/GrayscaleColorConversion.cs
  38. 178
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/JpegColorPacking.cs
  39. 117
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/JpegColorPackingScalar.cs
  40. 3
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/RgbColorConversion.cs
  41. 3
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/YCbCrColorConversion.cs
  42. 244
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/YCbCrOperatorAssembly.cs
  43. 3
      tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/YccKColorConverter.cs
  44. 64
      tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeAssembly.cs
  45. 175
      tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssembly.cs
  46. 128
      tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs
  47. 59
      tests/ImageSharp.Benchmarks/General/PixelConversion/Vector4AffineTransformAssembly.cs
  48. 327
      tests/ImageSharp.Benchmarks/PixelBlenders/PixelBlenderTraversalAssembly.cs
  49. 70
      tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs
  50. 105
      tests/ImageSharp.Tests/Formats/Jpg/JpegColorConverterTests.cs
  51. 168
      tests/ImageSharp.Tests/Formats/Jpg/JpegColorPackingTests.cs
  52. 79
      tests/ImageSharp.Tests/Formats/Png/PngEncoderFilterTests.cs
  53. 16
      tests/ImageSharp.Tests/PixelFormats/PixelBlenderTests.cs
  54. 32
      tests/ImageSharp.Tests/PixelFormats/Vector4ConvertersTests.cs

9
src/ImageSharp/ColorProfiles/ColorProfileConverterExtensionsIcc.cs

@ -660,6 +660,8 @@ internal static class ColorProfileConverterExtensionsIcc
private static void ClipNegative(Span<Vector4> source) private static void ClipNegative(Span<Vector4> source)
{ {
// Vector4 values are contiguous floats, so flattening preserves the component order
// while allowing one shared tensor traversal to process every channel and SIMD tail.
Span<float> values = MemoryMarshal.Cast<Vector4, float>(source); Span<float> values = MemoryMarshal.Cast<Vector4, float>(source);
TensorPrimitives_.Max(values, 0F, values); TensorPrimitives_.Max(values, 0F, values);
} }
@ -680,10 +682,9 @@ internal static class ColorProfileConverterExtensionsIcc
private static void LabToLab(Span<Vector4> source, Span<Vector4> destination, [ConstantExpected] float scale) private static void LabToLab(Span<Vector4> source, Span<Vector4> destination, [ConstantExpected] float scale)
{ {
TensorPrimitives_.Multiply( // Reinterpreting both spans exposes all four components to one multiplication traversal;
MemoryMarshal.Cast<Vector4, float>(source), // the source and destination retain their original Vector4 boundaries after the operation.
scale, TensorPrimitives_.Multiply(MemoryMarshal.Cast<Vector4, float>(source), scale, MemoryMarshal.Cast<Vector4, float>(destination));
MemoryMarshal.Cast<Vector4, float>(destination));
} }
private class ConversionParams private class ConversionParams

33
src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs

@ -1,8 +1,6 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Buffers.Binary;
using System.Numerics;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics; using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Common.Helpers;
@ -37,11 +35,18 @@ internal readonly struct WXYZPad3Shuffle4 : IPad3Shuffle4
{ {
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) => BitOperations.RotateLeft(source, 8); public static uint Invoke(uint source)
// The scalar pipeline has already appended opaque W, so the four-component
// WXYZ operator performs the complete remaining permutation.
=> WXYZShuffle4.Invoke(source);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is an XYZW pixel with opaque W. Selecting [3, 0, 1, 2]
// produces WXYZ, and offsets 4, 8, and 12 repeat that rotation for the next pixels.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14));
} }
@ -52,11 +57,18 @@ internal readonly struct WZYXPad3Shuffle4 : IPad3Shuffle4
{ {
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source); public static uint Invoke(uint source)
// The scalar pipeline has already appended opaque W, so the four-component
// WZYX operator performs the complete remaining permutation.
=> WZYXShuffle4.Invoke(source);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is an XYZW pixel with opaque W. Selecting [3, 2, 1, 0]
// produces WZYX, and offsets 4, 8, and 12 repeat that reversal for the next pixels.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
} }
@ -68,15 +80,16 @@ internal readonly struct ZYXWPad3Shuffle4 : IPad3Shuffle4
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// Preserve opaque W and Y while exchanging X and Z. // The scalar pipeline has already appended opaque W, so the four-component
uint wy = source & 0xFF00FF00; // ZYXW operator performs the complete remaining permutation.
uint xz = source & 0x00FF00FF; => ZYXWShuffle4.Invoke(source);
return wy | BitOperations.RotateLeft(xz, 16);
}
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is an XYZW pixel with opaque W. Selecting [2, 1, 0, 3]
// exchanges X and Z to produce ZYXW, with offsets 4, 8, and 12 covering the next pixels.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
} }

15
src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs

@ -22,16 +22,17 @@ internal readonly struct ZYXShuffle3 : IShuffle3
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// Y is already centered; shift X and Z directly into each other's byte positions. // The scalar tail is staged as XYZW with an unused W byte. Reusing the four-component
uint y = source & 0x0000FF00; // ZYXW operator produces ZYX in the low three bytes consumed by the caller.
uint x = (source & 0x000000FF) << 16; => ZYXWShuffle4.Invoke(source);
uint z = (source & 0x00FF0000) >> 16;
return x | y | z;
}
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is a temporary XYZW pixel created by the shuffle pipeline.
// Selecting [2, 1, 0, 3] produces ZYXW, and offsets 4, 8, and 12 repeat that
// permutation for the next pixels. The pipeline subsequently discards every W byte.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
} }

219
src/ImageSharp/Common/Helpers/Shuffle/IShuffle4.cs

@ -27,6 +27,30 @@ internal interface IShuffle4 : IComponentShuffle
/// <param name="source">The source pixels.</param> /// <param name="source">The source pixels.</param>
/// <returns>The reordered pixels.</returns> /// <returns>The reordered pixels.</returns>
public static abstract Vector512<byte> Invoke(Vector512<byte> source); public static abstract Vector512<byte> Invoke(Vector512<byte> source);
/// <summary>
/// Expands one 128-bit lane mask into absolute indices for a 512-bit shuffle.
/// </summary>
/// <param name="laneMask">The indices, from zero through fifteen, for one 128-bit lane.</param>
/// <returns>The corresponding absolute indices for all four 128-bit lanes.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> ExpandLaneMask(Vector128<byte> laneMask)
{
// A 512-bit vector contains four 128-bit lanes, and each lane contains four packed
// XYZW pixels. The supplied mask addresses bytes 0..15 in the first lane. The managed
// Vector512.Shuffle fallback addresses the complete 64-byte vector, so the same
// permutation must address bytes 16..31, 32..47, and 48..63 in the remaining lanes.
//
// AVX-512BW VPSHUFB instead interprets indices independently within each 128-bit lane
// and uses only the low four bits to select a byte. Adding the lane offsets therefore
// satisfies the managed absolute-index contract without changing the native lane-local
// permutation.
Vector128<byte> lane1 = laneMask + Vector128.Create((byte)16);
Vector128<byte> lane2 = laneMask + Vector128.Create((byte)32);
Vector128<byte> lane3 = laneMask + Vector128.Create((byte)48);
return Vector512.Create(Vector256.Create(laneMask, lane1), Vector256.Create(lane2, lane3));
}
} }
/// <summary> /// <summary>
@ -37,58 +61,42 @@ internal readonly struct WXYZShuffle4 : IShuffle4
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// source = [W Z Y X] // source = [W Z Y X]
// ROTL(8, source) = [Z Y X W] // ROTL(8, source) = [Z Y X W]
return BitOperations.RotateLeft(source, 8); => BitOperations.RotateLeft(source, 8);
}
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask()); => Vector128_.ShuffleNative(source, CreateLaneMask());
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source) public static Vector256<byte> Invoke(Vector256<byte> source)
{ {
// AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices. // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
Vector128<byte> mask = CreateMask(); Vector128<byte> mask = CreateLaneMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
} }
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source) public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
// Expand the four-pixel lane permutation across all four 128-bit lanes.
=> Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask()));
/// <summary> /// <summary>
/// Creates the indices that rotate each XYZW pixel to WXYZ within one 128-bit lane. /// Creates the indices that rotate each XYZW pixel to WXYZ within one 128-bit lane.
/// </summary> /// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns> /// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask() private static Vector128<byte> CreateLaneMask()
=> Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14);
/// <summary> // Each four-byte group is one XYZW pixel. Selecting [3, 0, 1, 2] produces
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. // WXYZ, and offsets 4, 8, and 12 repeat that permutation for the next pixels.
/// </summary> => Vector128.Create((byte)3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14);
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
{
// Native vpshufb ignores the lane offsets, while Vector512.Shuffle treats the indices as
// absolute. Encoding both meanings keeps the AVX-512 and managed fallback results identical.
return Vector512.Create(
0x0605040702010003UL,
0x0E0D0C0F0A09080BUL,
0x1615141712111013UL,
0x1E1D1C1F1A19181BUL,
0x2625242722212023UL,
0x2E2D2C2F2A29282BUL,
0x3635343732313033UL,
0x3E3D3C3F3A39383BUL).AsByte();
}
} }
/// <summary> /// <summary>
@ -99,52 +107,42 @@ internal readonly struct WZYXShuffle4 : IShuffle4
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// Reversing the integer's endianness also reverses the four byte components. // source = [W Z Y X]
return BinaryPrimitives.ReverseEndianness(source); // REVERSE(source) = [X Y Z W]
} => BinaryPrimitives.ReverseEndianness(source);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask()); => Vector128_.ShuffleNative(source, CreateLaneMask());
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source) public static Vector256<byte> Invoke(Vector256<byte> source)
{ {
Vector128<byte> mask = CreateMask(); // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
Vector128<byte> mask = CreateLaneMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
} }
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source) public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
// Expand the four-pixel lane permutation across all four 128-bit lanes.
=> Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask()));
/// <summary> /// <summary>
/// Creates the indices that reverse each XYZW pixel to WZYX within one 128-bit lane. /// Creates the indices that reverse each XYZW pixel to WZYX within one 128-bit lane.
/// </summary> /// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns> /// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask() private static Vector128<byte> CreateLaneMask()
=> Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12);
/// <summary> // Each four-byte group is one XYZW pixel. Selecting [3, 2, 1, 0] produces
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. // WZYX, and offsets 4, 8, and 12 repeat that reversal for the next pixels.
/// </summary> => Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12);
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0405060700010203UL,
0x0C0D0E0F08090A0BUL,
0x1415161710111213UL,
0x1C1D1E1F18191A1BUL,
0x2425262720212223UL,
0x2C2D2E2F28292A2BUL,
0x3435363730313233UL,
0x3C3D3E3F38393A3BUL).AsByte();
} }
/// <summary> /// <summary>
@ -155,53 +153,42 @@ internal readonly struct YZWXShuffle4 : IShuffle4
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// source = [W Z Y X] // source = [W Z Y X]
// ROTR(8, source) = [X W Z Y] // ROTR(8, source) = [X W Z Y]
return BitOperations.RotateRight(source, 8); => BitOperations.RotateRight(source, 8);
}
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask()); => Vector128_.ShuffleNative(source, CreateLaneMask());
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source) public static Vector256<byte> Invoke(Vector256<byte> source)
{ {
Vector128<byte> mask = CreateMask(); // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
Vector128<byte> mask = CreateLaneMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
} }
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source) public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
// Expand the four-pixel lane permutation across all four 128-bit lanes.
=> Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask()));
/// <summary> /// <summary>
/// Creates the indices that rotate each XYZW pixel to YZWX within one 128-bit lane. /// Creates the indices that rotate each XYZW pixel to YZWX within one 128-bit lane.
/// </summary> /// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns> /// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask() private static Vector128<byte> CreateLaneMask()
=> Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12);
/// <summary> // Each four-byte group is one XYZW pixel. Selecting [1, 2, 3, 0] produces
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. // YZWX, and offsets 4, 8, and 12 repeat that rotation for the next pixels.
/// </summary> => Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12);
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0407060500030201UL,
0x0C0F0E0D080B0A09UL,
0x1417161510131211UL,
0x1C1F1E1D181B1A19UL,
0x2427262520232221UL,
0x2C2F2E2D282B2A29UL,
0x3437363530333231UL,
0x3C3F3E3D383B3A39UL).AsByte();
} }
/// <summary> /// <summary>
@ -212,54 +199,44 @@ internal readonly struct ZYXWShuffle4 : IShuffle4
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// Preserve W and Y while rotating the masked X/Z bytes into each other's positions. // source = [W Z Y X]
uint wy = source & 0xFF00FF00; // source & 0xFF00FF00 = [W 0 Y 0]
uint xz = source & 0x00FF00FF; // ROTL(source & 0x00FF00FF) = [0 X 0 Z]
return wy | BitOperations.RotateLeft(xz, 16); // combined = [W X Y Z]
} => (source & 0xFF00FF00) | BitOperations.RotateLeft(source & 0x00FF00FF, 16);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask()); => Vector128_.ShuffleNative(source, CreateLaneMask());
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source) public static Vector256<byte> Invoke(Vector256<byte> source)
{ {
Vector128<byte> mask = CreateMask(); // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
Vector128<byte> mask = CreateLaneMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
} }
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source) public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
// Expand the four-pixel lane permutation across all four 128-bit lanes.
=> Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask()));
/// <summary> /// <summary>
/// Creates the indices that exchange X and Z in each XYZW pixel within one 128-bit lane. /// Creates the indices that exchange X and Z in each XYZW pixel within one 128-bit lane.
/// </summary> /// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns> /// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask() private static Vector128<byte> CreateLaneMask()
=> Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15);
/// <summary> // Each four-byte group is one XYZW pixel. Selecting [2, 1, 0, 3] exchanges
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. // X and Z to produce ZYXW, with offsets 4, 8, and 12 covering the next pixels.
/// </summary> => Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15);
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0704050603000102UL,
0x0F0C0D0E0B08090AUL,
0x1714151613101112UL,
0x1F1C1D1E1B18191AUL,
0x2724252623202122UL,
0x2F2C2D2E2B28292AUL,
0x3734353633303132UL,
0x3F3C3D3E3B38393AUL).AsByte();
} }
/// <summary> /// <summary>
@ -270,52 +247,42 @@ internal readonly struct XWZYShuffle4 : IShuffle4
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// Preserve X and Z while rotating the masked Y/W bytes into each other's positions. // source = [W Z Y X]
uint xz = source & 0x00FF00FF; // source & 0x00FF00FF = [0 Z 0 X]
uint yw = source & 0xFF00FF00; // ROTL(source & 0xFF00FF00) = [Y 0 W 0]
return xz | BitOperations.RotateLeft(yw, 16); // combined = [Y Z W X]
} => (source & 0x00FF00FF) | BitOperations.RotateLeft(source & 0xFF00FF00, 16);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
=> Vector128_.ShuffleNative(source, CreateMask()); => Vector128_.ShuffleNative(source, CreateLaneMask());
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<byte> Invoke(Vector256<byte> source) public static Vector256<byte> Invoke(Vector256<byte> source)
{ {
Vector128<byte> mask = CreateMask(); // AVX2 byte shuffles select within 128-bit lanes, so both halves use the same pixel-local indices.
Vector128<byte> mask = CreateLaneMask();
return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask)); return Vector256_.ShufflePerLane(source, Vector256.Create(mask, mask));
} }
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> Invoke(Vector512<byte> source) public static Vector512<byte> Invoke(Vector512<byte> source)
=> Vector512_.ShuffleNative(source, CreateMask512());
// Expand the four-pixel lane permutation across all four 128-bit lanes.
=> Vector512_.ShuffleNative(source, IShuffle4.ExpandLaneMask(CreateLaneMask()));
/// <summary> /// <summary>
/// Creates the indices that exchange Y and W in each XYZW pixel within one 128-bit lane. /// Creates the indices that exchange Y and W in each XYZW pixel within one 128-bit lane.
/// </summary> /// </summary>
/// <returns>The pixel-local byte shuffle indices.</returns> /// <returns>The pixel-local byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> CreateMask() private static Vector128<byte> CreateLaneMask()
=> Vector128.Create((byte)0, 3, 2, 1, 4, 7, 6, 5, 8, 11, 10, 9, 12, 15, 14, 13);
/// <summary> // Each four-byte group is one XYZW pixel. Selecting [0, 3, 2, 1] exchanges
/// Creates absolute indices for all four 128-bit lanes in a 512-bit vector. // Y and W to produce XWZY, with offsets 4, 8, and 12 covering the next pixels.
/// </summary> => Vector128.Create((byte)0, 3, 2, 1, 4, 7, 6, 5, 8, 11, 10, 9, 12, 15, 14, 13);
/// <returns>The absolute byte shuffle indices.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<byte> CreateMask512()
=> Vector512.Create(
0x0506070401020300UL,
0x0D0E0F0C090A0B08UL,
0x1516171411121310UL,
0x1D1E1F1C191A1B18UL,
0x2526272421222320UL,
0x2D2E2F2C292A2B28UL,
0x3536373431323330UL,
0x3D3E3F3C393A3B38UL).AsByte();
} }

39
src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs

@ -1,8 +1,6 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Buffers.Binary;
using System.Numerics;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Runtime.Intrinsics; using System.Runtime.Intrinsics;
@ -38,11 +36,19 @@ internal readonly struct YZWXShuffle4Slice3 : IShuffle4Slice3
{ {
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) => BitOperations.RotateRight(source, 8); public static uint Invoke(uint source)
// Reuse the four-component rotation; the caller stores only the low YZW
// bytes and therefore discards the rotated X byte.
=> YZWXShuffle4.Invoke(source);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is an XYZW pixel. Selecting [1, 2, 3, 0] produces
// YZWX, and offsets 4, 8, and 12 repeat that rotation for the next pixels.
// The surrounding pipeline subsequently removes every fourth byte.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12));
} }
@ -53,11 +59,19 @@ internal readonly struct WZYXShuffle4Slice3 : IShuffle4Slice3
{ {
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) => BinaryPrimitives.ReverseEndianness(source); public static uint Invoke(uint source)
// Reuse the four-component reversal; the caller stores only the low WZY
// bytes and therefore discards the reversed X byte.
=> WZYXShuffle4.Invoke(source);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is an XYZW pixel. Selecting [3, 2, 1, 0] produces
// WZYX, and offsets 4, 8, and 12 repeat that reversal for the next pixels.
// The surrounding pipeline subsequently removes every fourth byte.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12));
} }
@ -69,19 +83,24 @@ internal readonly struct ZYXWShuffle4Slice3 : IShuffle4Slice3
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.ShortMethod)] [MethodImpl(InliningOptions.ShortMethod)]
public static uint Invoke(uint source) public static uint Invoke(uint source)
{
// Preserve W and Y while exchanging X and Z; W is subsequently discarded. // Reuse the four-component exchange; the caller stores only the low ZYX
uint wy = source & 0xFF00FF00; // bytes and therefore discards the preserved W byte.
uint xz = source & 0x00FF00FF; => ZYXWShuffle4.Invoke(source);
return wy | BitOperations.RotateLeft(xz, 16);
}
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<byte> Invoke(Vector128<byte> source) public static Vector128<byte> Invoke(Vector128<byte> source)
// Each four-byte group is an XYZW pixel. Selecting [2, 1, 0, 3] produces
// ZYXW, and offsets 4, 8, and 12 repeat that exchange for the next pixels.
// The surrounding pipeline subsequently removes every fourth byte.
=> Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15)); => Vector128_.ShuffleNative(source, Vector128.Create((byte)2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
} }
/// <summary>
/// Represents one tightly packed three-byte value for scalar four-to-three component writes.
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 3)] [StructLayout(LayoutKind.Explicit, Size = 3)]
internal readonly struct Byte3 internal readonly struct Byte3
{ {

94
src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs

@ -67,22 +67,17 @@ internal static partial class SimdUtils
{ {
// Four independent vectors amortize loop control and expose enough work for the CPU // Four independent vectors amortize loop control and expose enough work for the CPU
// to overlap loads, byte shuffles, and stores without changing pixel ordering. // to overlap loads, byte shuffles, and stores without changing pixel ordering.
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i)) TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i)).StoreUnsafe(ref destinationBase, (nuint)i);
.StoreUnsafe(ref destinationBase, (nuint)i); TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + Vector512<byte>.Count))).StoreUnsafe(ref destinationBase, (nuint)(i + Vector512<byte>.Count));
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + Vector512<byte>.Count))) TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512<byte>.Count * 2)))).StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512<byte>.Count * 2)));
.StoreUnsafe(ref destinationBase, (nuint)(i + Vector512<byte>.Count)); TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512<byte>.Count * 3)))).StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512<byte>.Count * 3)));
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512<byte>.Count * 2))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512<byte>.Count * 2)));
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector512<byte>.Count * 3))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector512<byte>.Count * 3)));
} }
int oneVectorFromEnd = length - Vector512<byte>.Count; int oneVectorFromEnd = length - Vector512<byte>.Count;
for (; i <= oneVectorFromEnd; i += Vector512<byte>.Count) for (; i <= oneVectorFromEnd; i += Vector512<byte>.Count)
{ {
TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i)) TShuffle.Invoke(Vector512.LoadUnsafe(ref sourceBase, (nuint)i)).StoreUnsafe(ref destinationBase, (nuint)i);
.StoreUnsafe(ref destinationBase, (nuint)i);
} }
} }
@ -92,22 +87,17 @@ internal static partial class SimdUtils
for (; i <= fourVectorsFromEnd; i += Vector256<byte>.Count * 4) for (; i <= fourVectorsFromEnd; i += Vector256<byte>.Count * 4)
{ {
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i)) TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i)).StoreUnsafe(ref destinationBase, (nuint)i);
.StoreUnsafe(ref destinationBase, (nuint)i); TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + Vector256<byte>.Count))).StoreUnsafe(ref destinationBase, (nuint)(i + Vector256<byte>.Count));
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + Vector256<byte>.Count))) TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256<byte>.Count * 2)))).StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256<byte>.Count * 2)));
.StoreUnsafe(ref destinationBase, (nuint)(i + Vector256<byte>.Count)); TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256<byte>.Count * 3)))).StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256<byte>.Count * 3)));
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256<byte>.Count * 2))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256<byte>.Count * 2)));
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector256<byte>.Count * 3))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector256<byte>.Count * 3)));
} }
int oneVectorFromEnd = length - Vector256<byte>.Count; int oneVectorFromEnd = length - Vector256<byte>.Count;
for (; i <= oneVectorFromEnd; i += Vector256<byte>.Count) for (; i <= oneVectorFromEnd; i += Vector256<byte>.Count)
{ {
TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i)) TShuffle.Invoke(Vector256.LoadUnsafe(ref sourceBase, (nuint)i)).StoreUnsafe(ref destinationBase, (nuint)i);
.StoreUnsafe(ref destinationBase, (nuint)i);
} }
} }
@ -117,22 +107,17 @@ internal static partial class SimdUtils
for (; i <= fourVectorsFromEnd; i += Vector128<byte>.Count * 4) for (; i <= fourVectorsFromEnd; i += Vector128<byte>.Count * 4)
{ {
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i)) TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i)).StoreUnsafe(ref destinationBase, (nuint)i);
.StoreUnsafe(ref destinationBase, (nuint)i); TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + Vector128<byte>.Count))).StoreUnsafe(ref destinationBase, (nuint)(i + Vector128<byte>.Count));
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + Vector128<byte>.Count))) TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128<byte>.Count * 2)))).StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128<byte>.Count * 2)));
.StoreUnsafe(ref destinationBase, (nuint)(i + Vector128<byte>.Count)); TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128<byte>.Count * 3)))).StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128<byte>.Count * 3)));
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128<byte>.Count * 2))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128<byte>.Count * 2)));
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)(i + (Vector128<byte>.Count * 3))))
.StoreUnsafe(ref destinationBase, (nuint)(i + (Vector128<byte>.Count * 3)));
} }
int oneVectorFromEnd = length - Vector128<byte>.Count; int oneVectorFromEnd = length - Vector128<byte>.Count;
for (; i <= oneVectorFromEnd; i += Vector128<byte>.Count) for (; i <= oneVectorFromEnd; i += Vector128<byte>.Count)
{ {
TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i)) TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)i)).StoreUnsafe(ref destinationBase, (nuint)i);
.StoreUnsafe(ref destinationBase, (nuint)i);
} }
} }
@ -167,15 +152,17 @@ internal static partial class SimdUtils
if (Vector128.IsHardwareAccelerated) if (Vector128.IsHardwareAccelerated)
{ {
// Each group contains sixteen XYZ pixels in three registers. The pad mask expands // Each group contains sixteen XYZ pixels in three registers. For a register beginning
// four triplets per register to XYZW, with 0x80 selecting zero for the temporary W lane. // [X0,Y0,Z0,X1,Y1,Z1,...], the indices [0,1,2,0x80,3,4,5,0x80,...]
Vector128<byte> padMask = Vector128.Create( // produce four [X,Y,Z,0] pixels. Index 0x80 selects zero on the native byte-shuffle
(byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80); // instructions and on the portable helper, creating a temporary W lane for TShuffle.
Vector128<byte> padMask = Vector128.Create((byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
// After the operator has reordered padded pixels, these masks remove every temporary
// W lane and repack the four registers into three contiguous XYZ destination registers. // After TShuffle places the retained components in bytes 0..2 of each four-byte pixel,
Vector128<byte> sliceMask = Vector128.Create( // [0,1,2,4,5,6,8,9,10,12,13,14] packs four triplets into twelve bytes. Rotating that
(byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80); // mask by twelve positions moves the packed bytes four positions right. Alternating the
// two alignments lets AlignRight stitch four twelve-byte results into three full registers.
Vector128<byte> sliceMask = Vector128.Create((byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
Vector128<byte> sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12); Vector128<byte> sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12);
ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase); ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase);
ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase); ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase);
@ -284,10 +271,13 @@ internal static partial class SimdUtils
if (Vector128.IsHardwareAccelerated) if (Vector128.IsHardwareAccelerated)
{ {
// The fixed mask expands four XYZ triplets to four XYZW pixels. The zeroed W bytes // For source bytes [X0,Y0,Z0,X1,Y1,Z1,...], the indices
// are then filled with opaque alpha before the selected operator reorders each pixel. // [0,1,2,0x80,3,4,5,0x80,...] form four [X,Y,Z,0] pixels. The native
Vector128<byte> padMask = Vector128.Create( // and portable shuffle paths both interpret 0x80 as a zero-producing index.
(byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80); Vector128<byte> padMask = Vector128.Create((byte)0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);
// The broadcast ulong has 0xFF in bytes 3 and 7; repeating it across 128 bits
// fills W at byte positions 3, 7, 11, and 15 without modifying X, Y, or Z.
Vector128<byte> opaqueAlpha = Vector128.Create(0xFF000000FF000000UL).AsByte(); Vector128<byte> opaqueAlpha = Vector128.Create(0xFF000000FF000000UL).AsByte();
ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase); ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase);
ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase); ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase);
@ -366,10 +356,14 @@ internal static partial class SimdUtils
if (Vector128.IsHardwareAccelerated) if (Vector128.IsHardwareAccelerated)
{ {
// Each operator first places the three retained components in the low bytes of every // Each operator first places the retained components in bytes 0..2 of every four-byte
// four-byte pixel. These masks then delete the fourth byte and compact sixteen pixels. // pixel. The indices [0,1,2,4,5,6,8,9,10,12,13,14] delete each fourth byte and
Vector128<byte> sliceMask = Vector128.Create( // pack four triplets into the low twelve bytes. Indices 0x80 zero the unused bytes.
(byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80); Vector128<byte> sliceMask = Vector128.Create((byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);
// Rotating the mask by twelve moves its packed result four bytes right. Alternating
// shifted and unshifted results gives AlignRight the overlap needed to concatenate
// four twelve-byte groups into three complete destination registers.
Vector128<byte> sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12); Vector128<byte> sliceEndMask = Vector128_.AlignRight(sliceMask, sliceMask, 12);
ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase); ref Vector128<byte> sourceVectors = ref Unsafe.As<byte, Vector128<byte>>(ref sourceBase);
ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase); ref Vector128<byte> destinationVectors = ref Unsafe.As<byte, Vector128<byte>>(ref destinationBase);
@ -377,8 +371,7 @@ internal static partial class SimdUtils
nuint sourceVectorIndex = 0; nuint sourceVectorIndex = 0;
nuint destinationVectorIndex = 0; nuint destinationVectorIndex = 0;
for (; sourceVectorIndex + 3 < sourceVectorCount; for (; sourceVectorIndex + 3 < sourceVectorCount; sourceVectorIndex += 4, destinationVectorIndex += 3)
sourceVectorIndex += 4, destinationVectorIndex += 3)
{ {
// Load and transform all sixteen source pixels before writing the shorter output group. // Load and transform all sixteen source pixels before writing the shorter output group.
// This preserves forward progress when source and destination begin at the same address. // This preserves forward progress when source and destination begin at the same address.
@ -413,8 +406,7 @@ internal static partial class SimdUtils
{ {
// The operator arranges the three retained components at the front of each pixel. // The operator arranges the three retained components at the front of each pixel.
// One fixed shuffle then compacts four pixels into the low twelve vector bytes. // One fixed shuffle then compacts four pixels into the low twelve vector bytes.
Vector128<byte> result = TShuffle.Invoke( Vector128<byte> result = TShuffle.Invoke(Vector128.LoadUnsafe(ref sourceBase, (nuint)sourceOffset));
Vector128.LoadUnsafe(ref sourceBase, (nuint)sourceOffset));
result = Vector128_.ShuffleNative(result, sliceMask); result = Vector128_.ShuffleNative(result, sliceMask);

19
src/ImageSharp/Common/Helpers/Vector128Utilities.cs

@ -1296,22 +1296,9 @@ internal static class Vector128_
return PackedSimd.SubtractSaturate(left, right); return PackedSimd.SubtractSaturate(left, right);
} }
// Widen inputs to 16-bit // Subtracting the smaller operand implements the .NET 10 unsigned contract:
(Vector128<ushort> leftLo, Vector128<ushort> leftHi) = Vector128.Widen(left); // lanes where right exceeds left subtract left from itself and therefore saturate at zero.
(Vector128<ushort> rightLo, Vector128<ushort> rightHi) = Vector128.Widen(right); return left - Vector128.Min(left, right);
// Subtract
Vector128<ushort> diffLo = leftLo - rightLo;
Vector128<ushort> diffHi = leftHi - rightHi;
// Clamp to signed 8-bit range
Vector128<ushort> max = Vector128.Create((ushort)byte.MaxValue);
diffLo = Clamp(diffLo, Vector128<ushort>.Zero, max);
diffHi = Clamp(diffHi, Vector128<ushort>.Zero, max);
// Narrow back to bytes
return Vector128.Narrow(diffLo, diffHi);
} }
/// <summary> /// <summary>

6
src/ImageSharp/Common/Helpers/Vector256Utilities.cs

@ -495,9 +495,9 @@ internal static class Vector256_
return Avx2.SubtractSaturate(left, right); return Avx2.SubtractSaturate(left, right);
} }
return Vector256.Create( // The .NET 10 portable implementation applies the same saturated operation to
Vector128_.SubtractSaturate(left.GetLower(), right.GetLower()), // both 128-bit halves, allowing each half to select its native instruction set.
Vector128_.SubtractSaturate(left.GetUpper(), right.GetUpper())); return Vector256.Create(Vector128_.SubtractSaturate(left.GetLower(), right.GetLower()), Vector128_.SubtractSaturate(left.GetUpper(), right.GetUpper()));
} }
/// <summary> /// <summary>

20
src/ImageSharp/Common/Helpers/Vector512Utilities.cs

@ -129,6 +129,26 @@ internal static class Vector512_
return Vector512.Create(lower, upper); return Vector512.Create(lower, upper);
} }
/// <summary>
/// Subtracts packed unsigned 8-bit integers in <paramref name="right"/> from
/// <paramref name="left"/>, saturating negative lane results to zero.
/// </summary>
/// <param name="left">The vector from which <paramref name="right"/> is subtracted.</param>
/// <param name="right">The vector to subtract from <paramref name="left"/>.</param>
/// <returns>The element-wise saturated differences.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<byte> SubtractSaturate(Vector512<byte> left, Vector512<byte> right)
{
if (Avx512BW.IsSupported)
{
return Avx512BW.SubtractSaturate(left, right);
}
// This mirrors the .NET 10 portable implementation: recursively processing both
// 256-bit halves preserves lane order and lets each half select its available ISA.
return Vector512.Create(Vector256_.SubtractSaturate(left.GetLower(), right.GetLower()), Vector256_.SubtractSaturate(left.GetUpper(), right.GetUpper()));
}
/// <summary> /// <summary>
/// Performs a multiplication and a negated addition of the <see cref="Vector512{Single}"/>. /// Performs a multiplication and a negated addition of the <see cref="Vector512{Single}"/>.
/// </summary> /// </summary>

90
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.CmykOperator.cs

@ -27,14 +27,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref float c0, ref float c1, ref float c2, float c3, float maximumValue, float halfValue, float scale)
ref float c0,
ref float c1,
ref float c2,
float c3,
float maximumValue,
float halfValue,
float scale)
{ {
// Adobe-style CMYK stores inverted component samples. Multiplying K by scale twice folds the // Adobe-style CMYK stores inverted component samples. Multiplying K by scale twice folds the
// two sample-domain divisions into one factor before it modulates the C, M, and Y planes. // two sample-domain divisions into one factor before it modulates the C, M, and Y planes.
@ -46,14 +39,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector128<float> c0, ref Vector128<float> c1, ref Vector128<float> c2, Vector128<float> c3, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale)
ref Vector128<float> c0,
ref Vector128<float> c1,
ref Vector128<float> c2,
Vector128<float> c3,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale)
{ {
// Each K lane supplies the common modulation factor for the corresponding C, M, and Y lanes. // Each K lane supplies the common modulation factor for the corresponding C, M, and Y lanes.
Vector128<float> scaledK = c3 * scale * scale; Vector128<float> scaledK = c3 * scale * scale;
@ -64,14 +50,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector256<float> c0, ref Vector256<float> c1, ref Vector256<float> c2, Vector256<float> c3, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale)
ref Vector256<float> c0,
ref Vector256<float> c1,
ref Vector256<float> c2,
Vector256<float> c3,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale)
{ {
// Eight independent CMYK samples remain lane-aligned throughout the modulation. // Eight independent CMYK samples remain lane-aligned throughout the modulation.
Vector256<float> scaledK = c3 * scale * scale; Vector256<float> scaledK = c3 * scale * scale;
@ -82,14 +61,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector512<float> c0, ref Vector512<float> c1, ref Vector512<float> c2, Vector512<float> c3, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale)
ref Vector512<float> c0,
ref Vector512<float> c1,
ref Vector512<float> c2,
Vector512<float> c3,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale)
{ {
// Sixteen independent CMYK samples remain lane-aligned throughout the modulation. // Sixteen independent CMYK samples remain lane-aligned throughout the modulation.
Vector512<float> scaledK = c3 * scale * scale; Vector512<float> scaledK = c3 * scale * scale;
@ -100,17 +72,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(float r, float g, float b, float maximumValue, float halfValue, float scale, out float c0, out float c1, out float c2, out float c3)
float r,
float g,
float b,
float maximumValue,
float halfValue,
float scale,
out float c0,
out float c1,
out float c2,
out float c3)
{ {
float c = maximumValue - r; float c = maximumValue - r;
float m = maximumValue - g; float m = maximumValue - g;
@ -144,17 +106,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector128<float> r, Vector128<float> g, Vector128<float> b, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale, out Vector128<float> c0, out Vector128<float> c1, out Vector128<float> c2, out Vector128<float> c3)
Vector128<float> r,
Vector128<float> g,
Vector128<float> b,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3)
{ {
Vector128<float> c = maximumValue - r; Vector128<float> c = maximumValue - r;
Vector128<float> m = maximumValue - g; Vector128<float> m = maximumValue - g;
@ -176,17 +128,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector256<float> r, Vector256<float> g, Vector256<float> b, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale, out Vector256<float> c0, out Vector256<float> c1, out Vector256<float> c2, out Vector256<float> c3)
Vector256<float> r,
Vector256<float> g,
Vector256<float> b,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3)
{ {
Vector256<float> c = maximumValue - r; Vector256<float> c = maximumValue - r;
Vector256<float> m = maximumValue - g; Vector256<float> m = maximumValue - g;
@ -208,17 +150,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector512<float> r, Vector512<float> g, Vector512<float> b, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale, out Vector512<float> c0, out Vector512<float> c1, out Vector512<float> c2, out Vector512<float> c3)
Vector512<float> r,
Vector512<float> g,
Vector512<float> b,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3)
{ {
Vector512<float> c = maximumValue - r; Vector512<float> c = maximumValue - r;
Vector512<float> m = maximumValue - g; Vector512<float> m = maximumValue - g;
@ -240,11 +172,7 @@ internal abstract partial class JpegColorConverterBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public static void ConvertToRgbInPlaceWithIcc( public static void ConvertToRgbInPlaceWithIcc(Configuration configuration, IccProfile profile, in ComponentValues values, float maximumValue)
Configuration configuration,
IccProfile profile,
in ComponentValues values,
float maximumValue)
{ {
using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 4); using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 4);
Span<float> packed = memoryOwner.Memory.Span; Span<float> packed = memoryOwner.Memory.Span;

105
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.GrayScaleOperator.cs

@ -28,14 +28,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref float c0, ref float c1, ref float c2, float c3, float maximumValue, float halfValue, float scale)
ref float c0,
ref float c1,
ref float c2,
float c3,
float maximumValue,
float halfValue,
float scale)
{ {
// JPEG stores luminance in the integer sample domain. Normalize it once, then duplicate the // JPEG stores luminance in the integer sample domain. Normalize it once, then duplicate the
// same value into all three RGB planes. Keeping it local also prevents potentially aliasing // same value into all three RGB planes. Keeping it local also prevents potentially aliasing
@ -48,14 +41,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector128<float> c0, ref Vector128<float> c1, ref Vector128<float> c2, Vector128<float> c3, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale)
ref Vector128<float> c0,
ref Vector128<float> c1,
ref Vector128<float> c2,
Vector128<float> c3,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale)
{ {
// Each XMM lane is one independent luminance sample. Reusing the normalized vector for R, G, // Each XMM lane is one independent luminance sample. Reusing the normalized vector for R, G,
// and B avoids recomputing the scale and keeps it live across potentially aliasing byref stores. // and B avoids recomputing the scale and keeps it live across potentially aliasing byref stores.
@ -67,14 +53,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector256<float> c0, ref Vector256<float> c1, ref Vector256<float> c2, Vector256<float> c3, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale)
ref Vector256<float> c0,
ref Vector256<float> c1,
ref Vector256<float> c2,
Vector256<float> c3,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale)
{ {
// Eight luminance samples occupy the YMM lanes. The local retains the normalized vector across // Eight luminance samples occupy the YMM lanes. The local retains the normalized vector across
// all three output stores even when the destination planes alias. // all three output stores even when the destination planes alias.
@ -86,14 +65,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector512<float> c0, ref Vector512<float> c1, ref Vector512<float> c2, Vector512<float> c3, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale)
ref Vector512<float> c0,
ref Vector512<float> c1,
ref Vector512<float> c2,
Vector512<float> c3,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale)
{ {
// Sixteen luminance samples occupy the ZMM lanes. The local retains the normalized vector across // Sixteen luminance samples occupy the ZMM lanes. The local retains the normalized vector across
// all three output stores without shuffles, interleaving, or source reloads. // all three output stores without shuffles, interleaving, or source reloads.
@ -105,17 +77,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(float r, float g, float b, float maximumValue, float halfValue, float scale, out float c0, out float c1, out float c2, out float c3)
float r,
float g,
float b,
float maximumValue,
float halfValue,
float scale,
out float c0,
out float c1,
out float c2,
out float c3)
{ {
// Rec.601 luma weights operate directly in the encoder sample domain. Only c0 is stored for a // Rec.601 luma weights operate directly in the encoder sample domain. Only c0 is stored for a
// one-component model; the remaining out values exist solely to satisfy the common operator shape. // one-component model; the remaining out values exist solely to satisfy the common operator shape.
@ -127,23 +89,10 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector128<float> r, Vector128<float> g, Vector128<float> b, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale, out Vector128<float> c0, out Vector128<float> c1, out Vector128<float> c2, out Vector128<float> c3)
Vector128<float> r,
Vector128<float> g,
Vector128<float> b,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3)
{ {
// The nested estimate gives each pixel the same multiply-add grouping as the scalar Rec.601 formula. // The nested estimate gives each pixel the same multiply-add grouping as the scalar Rec.601 formula.
c0 = Vector128_.MultiplyAddEstimate( c0 = Vector128_.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b));
Vector128.Create(0.299F),
r,
Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b));
c1 = default; c1 = default;
c2 = default; c2 = default;
c3 = default; c3 = default;
@ -151,23 +100,10 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector256<float> r, Vector256<float> g, Vector256<float> b, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale, out Vector256<float> c0, out Vector256<float> c1, out Vector256<float> c2, out Vector256<float> c3)
Vector256<float> r,
Vector256<float> g,
Vector256<float> b,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3)
{ {
// YMM lanes evaluate the same Rec.601 equation independently, with no horizontal lane reduction. // YMM lanes evaluate the same Rec.601 equation independently, with no horizontal lane reduction.
c0 = Vector256_.MultiplyAddEstimate( c0 = Vector256_.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b));
Vector256.Create(0.299F),
r,
Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b));
c1 = default; c1 = default;
c2 = default; c2 = default;
c3 = default; c3 = default;
@ -175,34 +111,17 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector512<float> r, Vector512<float> g, Vector512<float> b, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale, out Vector512<float> c0, out Vector512<float> c1, out Vector512<float> c2, out Vector512<float> c3)
Vector512<float> r,
Vector512<float> g,
Vector512<float> b,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3)
{ {
// ZMM lanes retain the same arithmetic order as narrower paths so only SIMD width changes. // ZMM lanes retain the same arithmetic order as narrower paths so only SIMD width changes.
c0 = Vector512_.MultiplyAddEstimate( c0 = Vector512_.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b));
Vector512.Create(0.299F),
r,
Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b));
c1 = default; c1 = default;
c2 = default; c2 = default;
c3 = default; c3 = default;
} }
/// <inheritdoc/> /// <inheritdoc/>
public static void ConvertToRgbInPlaceWithIcc( public static void ConvertToRgbInPlaceWithIcc(Configuration configuration, IccProfile profile, in ComponentValues values, float maximumValue)
Configuration configuration,
IccProfile profile,
in ComponentValues values,
float maximumValue)
{ {
using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 3); using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 3);
Span<float> packed = memoryOwner.Memory.Span; Span<float> packed = memoryOwner.Memory.Span;

167
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Operator.cs

@ -40,14 +40,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="maximumValue">The maximum component value for the configured precision.</param> /// <param name="maximumValue">The maximum component value for the configured precision.</param>
/// <param name="halfValue">The midpoint component value for the configured precision.</param> /// <param name="halfValue">The midpoint component value for the configured precision.</param>
/// <param name="scale">The reciprocal of <paramref name="maximumValue"/>.</param> /// <param name="scale">The reciprocal of <paramref name="maximumValue"/>.</param>
public static abstract void ConvertToRgb( public static abstract void ConvertToRgb(ref float c0, ref float c1, ref float c2, float c3, float maximumValue, float halfValue, float scale);
ref float c0,
ref float c1,
ref float c2,
float c3,
float maximumValue,
float halfValue,
float scale);
/// <summary> /// <summary>
/// Converts four JPEG samples to normalized RGB. /// Converts four JPEG samples to normalized RGB.
@ -59,14 +52,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="maximumValue">The maximum component value for the configured precision.</param> /// <param name="maximumValue">The maximum component value for the configured precision.</param>
/// <param name="halfValue">The midpoint component value for the configured precision.</param> /// <param name="halfValue">The midpoint component value for the configured precision.</param>
/// <param name="scale">The reciprocal of <paramref name="maximumValue"/> in every lane.</param> /// <param name="scale">The reciprocal of <paramref name="maximumValue"/> in every lane.</param>
public static abstract void ConvertToRgb( public static abstract void ConvertToRgb(ref Vector128<float> c0, ref Vector128<float> c1, ref Vector128<float> c2, Vector128<float> c3, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale);
ref Vector128<float> c0,
ref Vector128<float> c1,
ref Vector128<float> c2,
Vector128<float> c3,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale);
/// <summary> /// <summary>
/// Converts eight JPEG samples to normalized RGB. /// Converts eight JPEG samples to normalized RGB.
@ -78,14 +64,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="maximumValue">The maximum component value for the configured precision.</param> /// <param name="maximumValue">The maximum component value for the configured precision.</param>
/// <param name="halfValue">The midpoint component value for the configured precision.</param> /// <param name="halfValue">The midpoint component value for the configured precision.</param>
/// <param name="scale">The reciprocal of <paramref name="maximumValue"/> in every lane.</param> /// <param name="scale">The reciprocal of <paramref name="maximumValue"/> in every lane.</param>
public static abstract void ConvertToRgb( public static abstract void ConvertToRgb(ref Vector256<float> c0, ref Vector256<float> c1, ref Vector256<float> c2, Vector256<float> c3, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale);
ref Vector256<float> c0,
ref Vector256<float> c1,
ref Vector256<float> c2,
Vector256<float> c3,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale);
/// <summary> /// <summary>
/// Converts sixteen JPEG samples to normalized RGB. /// Converts sixteen JPEG samples to normalized RGB.
@ -97,14 +76,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="maximumValue">The maximum component value for the configured precision.</param> /// <param name="maximumValue">The maximum component value for the configured precision.</param>
/// <param name="halfValue">The midpoint component value for the configured precision.</param> /// <param name="halfValue">The midpoint component value for the configured precision.</param>
/// <param name="scale">The reciprocal of <paramref name="maximumValue"/> in every lane.</param> /// <param name="scale">The reciprocal of <paramref name="maximumValue"/> in every lane.</param>
public static abstract void ConvertToRgb( public static abstract void ConvertToRgb(ref Vector512<float> c0, ref Vector512<float> c1, ref Vector512<float> c2, Vector512<float> c3, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale);
ref Vector512<float> c0,
ref Vector512<float> c1,
ref Vector512<float> c2,
Vector512<float> c3,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale);
/// <summary> /// <summary>
/// Converts one RGB sample to JPEG components. /// Converts one RGB sample to JPEG components.
@ -119,17 +91,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="c1">The second converted component.</param> /// <param name="c1">The second converted component.</param>
/// <param name="c2">The third converted component.</param> /// <param name="c2">The third converted component.</param>
/// <param name="c3">The fourth converted component, if used.</param> /// <param name="c3">The fourth converted component, if used.</param>
public static abstract void ConvertFromRgb( public static abstract void ConvertFromRgb(float r, float g, float b, float maximumValue, float halfValue, float scale, out float c0, out float c1, out float c2, out float c3);
float r,
float g,
float b,
float maximumValue,
float halfValue,
float scale,
out float c0,
out float c1,
out float c2,
out float c3);
/// <summary> /// <summary>
/// Converts four RGB samples to JPEG components. /// Converts four RGB samples to JPEG components.
@ -144,17 +106,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="c1">The second converted component lanes.</param> /// <param name="c1">The second converted component lanes.</param>
/// <param name="c2">The third converted component lanes.</param> /// <param name="c2">The third converted component lanes.</param>
/// <param name="c3">The fourth converted component lanes, if used.</param> /// <param name="c3">The fourth converted component lanes, if used.</param>
public static abstract void ConvertFromRgb( public static abstract void ConvertFromRgb(Vector128<float> r, Vector128<float> g, Vector128<float> b, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale, out Vector128<float> c0, out Vector128<float> c1, out Vector128<float> c2, out Vector128<float> c3);
Vector128<float> r,
Vector128<float> g,
Vector128<float> b,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3);
/// <summary> /// <summary>
/// Converts eight RGB samples to JPEG components. /// Converts eight RGB samples to JPEG components.
@ -169,17 +121,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="c1">The second converted component lanes.</param> /// <param name="c1">The second converted component lanes.</param>
/// <param name="c2">The third converted component lanes.</param> /// <param name="c2">The third converted component lanes.</param>
/// <param name="c3">The fourth converted component lanes, if used.</param> /// <param name="c3">The fourth converted component lanes, if used.</param>
public static abstract void ConvertFromRgb( public static abstract void ConvertFromRgb(Vector256<float> r, Vector256<float> g, Vector256<float> b, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale, out Vector256<float> c0, out Vector256<float> c1, out Vector256<float> c2, out Vector256<float> c3);
Vector256<float> r,
Vector256<float> g,
Vector256<float> b,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3);
/// <summary> /// <summary>
/// Converts sixteen RGB samples to JPEG components. /// Converts sixteen RGB samples to JPEG components.
@ -194,17 +136,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="c1">The second converted component lanes.</param> /// <param name="c1">The second converted component lanes.</param>
/// <param name="c2">The third converted component lanes.</param> /// <param name="c2">The third converted component lanes.</param>
/// <param name="c3">The fourth converted component lanes, if used.</param> /// <param name="c3">The fourth converted component lanes, if used.</param>
public static abstract void ConvertFromRgb( public static abstract void ConvertFromRgb(Vector512<float> r, Vector512<float> g, Vector512<float> b, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale, out Vector512<float> c0, out Vector512<float> c1, out Vector512<float> c2, out Vector512<float> c3);
Vector512<float> r,
Vector512<float> g,
Vector512<float> b,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3);
/// <summary> /// <summary>
/// Converts JPEG component values to RGB using the supplied ICC profile. /// Converts JPEG component values to RGB using the supplied ICC profile.
@ -213,11 +145,7 @@ internal abstract partial class JpegColorConverterBase
/// <param name="profile">The source ICC profile.</param> /// <param name="profile">The source ICC profile.</param>
/// <param name="values">The component values to convert.</param> /// <param name="values">The component values to convert.</param>
/// <param name="maximumValue">The maximum component value for the configured precision.</param> /// <param name="maximumValue">The maximum component value for the configured precision.</param>
public static abstract void ConvertToRgbInPlaceWithIcc( public static abstract void ConvertToRgbInPlaceWithIcc(Configuration configuration, IccProfile profile, in ComponentValues values, float maximumValue);
Configuration configuration,
IccProfile profile,
in ComponentValues values,
float maximumValue);
} }
/// <summary> /// <summary>
@ -241,13 +169,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
public override int ElementsPerBatch public override int ElementsPerBatch
=> Vector512.IsHardwareAccelerated => Vector512.IsHardwareAccelerated ? Vector512<float>.Count : Vector256.IsHardwareAccelerated ? Vector256<float>.Count : Vector128.IsHardwareAccelerated ? Vector128<float>.Count : 1;
? Vector512<float>.Count
: Vector256.IsHardwareAccelerated
? Vector256<float>.Count
: Vector128.IsHardwareAccelerated
? Vector128<float>.Count
: 1;
/// <inheritdoc/> /// <inheritdoc/>
public override void ConvertToRgbInPlace(in ComponentValues values) public override void ConvertToRgbInPlace(in ComponentValues values)
@ -289,9 +211,7 @@ internal abstract partial class JpegColorConverterBase
// ComponentCount is a static property on the closed operator type, so the JIT removes // ComponentCount is a static property on the closed operator type, so the JIT removes
// this choice. Three-component models never dereference the empty Component3 byref. // this choice. Three-component models never dereference the empty Component3 byref.
Vector512<float> c3 = TOperator.ComponentCount == 4 Vector512<float> c3 = TOperator.ComponentCount == 4 ? Unsafe.As<float, Vector512<float>>(ref Unsafe.Add(ref c3Base, i)) : default;
? Unsafe.As<float, Vector512<float>>(ref Unsafe.Add(ref c3Base, i))
: default;
// c0-c2 alias the planar source vectors and are replaced in place with normalized RGB. // c0-c2 alias the planar source vectors and are replaced in place with normalized RGB.
// c3 is passed by value because the fourth JPEG component must remain unchanged. // c3 is passed by value because the fourth JPEG component must remain unchanged.
@ -321,9 +241,7 @@ internal abstract partial class JpegColorConverterBase
// The closed operator makes this a compile-time color-model choice, not a per-vector // The closed operator makes this a compile-time color-model choice, not a per-vector
// runtime abstraction or interface dispatch. // runtime abstraction or interface dispatch.
Vector256<float> c3 = TOperator.ComponentCount == 4 Vector256<float> c3 = TOperator.ComponentCount == 4 ? Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref c3Base, i)) : default;
? Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref c3Base, i))
: default;
TOperator.ConvertToRgb(ref c0, ref c1, ref c2, c3, maximumValue, halfValue, scaleVector); TOperator.ConvertToRgb(ref c0, ref c1, ref c2, c3, maximumValue, halfValue, scaleVector);
} }
@ -350,9 +268,7 @@ internal abstract partial class JpegColorConverterBase
ref Vector128<float> c2 = ref Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref c2Base, i)); ref Vector128<float> c2 = ref Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref c2Base, i));
// As at the wider stages, the fourth vector is loaded only for CMYK-shaped operators. // As at the wider stages, the fourth vector is loaded only for CMYK-shaped operators.
Vector128<float> c3 = TOperator.ComponentCount == 4 Vector128<float> c3 = TOperator.ComponentCount == 4 ? Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref c3Base, i)) : default;
? Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref c3Base, i))
: default;
TOperator.ConvertToRgb(ref c0, ref c1, ref c2, c3, maximumValue, halfValue, scaleVector); TOperator.ConvertToRgb(ref c0, ref c1, ref c2, c3, maximumValue, halfValue, scaleVector);
} }
@ -365,14 +281,7 @@ internal abstract partial class JpegColorConverterBase
{ {
float c3 = TOperator.ComponentCount == 4 ? Unsafe.Add(ref c3Base, i) : 0; float c3 = TOperator.ComponentCount == 4 ? Unsafe.Add(ref c3Base, i) : 0;
TOperator.ConvertToRgb( TOperator.ConvertToRgb(ref Unsafe.Add(ref c0Base, i), ref Unsafe.Add(ref c1Base, i), ref Unsafe.Add(ref c2Base, i), c3, this.MaximumValue, this.HalfValue, scale);
ref Unsafe.Add(ref c0Base, i),
ref Unsafe.Add(ref c1Base, i),
ref Unsafe.Add(ref c2Base, i),
c3,
this.MaximumValue,
this.HalfValue,
scale);
} }
} }
@ -420,17 +329,7 @@ internal abstract partial class JpegColorConverterBase
Vector512<float> g = Unsafe.As<float, Vector512<float>>(ref Unsafe.Add(ref gBase, i)); Vector512<float> g = Unsafe.As<float, Vector512<float>>(ref Unsafe.Add(ref gBase, i));
Vector512<float> b = Unsafe.As<float, Vector512<float>>(ref Unsafe.Add(ref bBase, i)); Vector512<float> b = Unsafe.As<float, Vector512<float>>(ref Unsafe.Add(ref bBase, i));
TOperator.ConvertFromRgb( TOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scaleVector, out Vector512<float> c0, out Vector512<float> c1, out Vector512<float> c2, out Vector512<float> c3);
r,
g,
b,
maximumValue,
halfValue,
scaleVector,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3);
// Outputs remain planar: each vector contains sixteen consecutive samples from one // Outputs remain planar: each vector contains sixteen consecutive samples from one
// JPEG component. Static count checks prevent grayscale from touching absent planes // JPEG component. Static count checks prevent grayscale from touching absent planes
@ -473,17 +372,7 @@ internal abstract partial class JpegColorConverterBase
Vector256<float> g = Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref gBase, i)); Vector256<float> g = Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref gBase, i));
Vector256<float> b = Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref bBase, i)); Vector256<float> b = Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref bBase, i));
TOperator.ConvertFromRgb( TOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scaleVector, out Vector256<float> c0, out Vector256<float> c1, out Vector256<float> c2, out Vector256<float> c3);
r,
g,
b,
maximumValue,
halfValue,
scaleVector,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3);
// Static count checks write only planes owned by this color model. // Static count checks write only planes owned by this color model.
Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref c0Base, i)) = c0; Unsafe.As<float, Vector256<float>>(ref Unsafe.Add(ref c0Base, i)) = c0;
@ -524,17 +413,7 @@ internal abstract partial class JpegColorConverterBase
Vector128<float> g = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref gBase, i)); Vector128<float> g = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref gBase, i));
Vector128<float> b = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref bBase, i)); Vector128<float> b = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref bBase, i));
TOperator.ConvertFromRgb( TOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scaleVector, out Vector128<float> c0, out Vector128<float> c1, out Vector128<float> c2, out Vector128<float> c3);
r,
g,
b,
maximumValue,
halfValue,
scaleVector,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3);
// Four results are stored only for the planes represented by the closed operator. // Four results are stored only for the planes represented by the closed operator.
Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref c0Base, i)) = c0; Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref c0Base, i)) = c0;
@ -560,17 +439,7 @@ internal abstract partial class JpegColorConverterBase
// Scalar conversion is reserved for the zero-to-three samples that cannot fill Vector128. // Scalar conversion is reserved for the zero-to-three samples that cannot fill Vector128.
for (; i < length; i++) for (; i < length; i++)
{ {
TOperator.ConvertFromRgb( TOperator.ConvertFromRgb(Unsafe.Add(ref rBase, i), Unsafe.Add(ref gBase, i), Unsafe.Add(ref bBase, i), this.MaximumValue, this.HalfValue, scale, out float c0, out float c1, out float c2, out float c3);
Unsafe.Add(ref rBase, i),
Unsafe.Add(ref gBase, i),
Unsafe.Add(ref bBase, i),
this.MaximumValue,
this.HalfValue,
scale,
out float c0,
out float c1,
out float c2,
out float c3);
Unsafe.Add(ref c0Base, i) = c0; Unsafe.Add(ref c0Base, i) = c0;

305
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Packing.cs

@ -0,0 +1,305 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Formats.Jpeg.Components;
internal abstract partial class JpegColorConverterBase
{
/// <summary>
/// Normalizes three planar component lanes and interleaves them into packed XYZ values.
/// </summary>
/// <param name="xLane">The planar X components.</param>
/// <param name="yLane">The planar Y components.</param>
/// <param name="zLane">The planar Z components.</param>
/// <param name="packed">The destination ordered as consecutive XYZ triples.</param>
/// <param name="scale">The normalization factor applied to every component.</param>
public static void PackedNormalizeInterleave3(ReadOnlySpan<float> xLane, ReadOnlySpan<float> yLane, ReadOnlySpan<float> zLane, Span<float> packed, float scale)
{
DebugGuard.IsTrue(packed.Length % 3 == 0, "Packed length must be divisible by 3.");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
DebugGuard.MustBeLessThanOrEqualTo(packed.Length / 3, xLane.Length, nameof(packed));
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
int i = 0;
if (Vector128.IsHardwareAccelerated)
{
Vector128<float> scaleVector = Vector128.Create(scale);
int oneVectorFromEnd = xLane.Length - Vector128<float>.Count;
for (; i <= oneVectorFromEnd; i += Vector128<float>.Count)
{
// Each source vector contains four consecutive samples from one plane:
// x = [X0 X1 X2 X3]
// y = [Y0 Y1 Y2 Y3]
// z = [Z0 Z1 Z2 Z3]
// Shifting X by one sample supplies the value that follows each XYZ triple:
// shiftedX = [X1 X2 X3 0]
// The transpose therefore produces overlapping rows [Xn Yn Zn Xn+1].
// AlignRight joins those rows into three complete destination vectors, avoiding
// the scalar-sized stores that writing four independent Vector3 values requires.
Vector128<float> x = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref xLaneRef, i)) * scaleVector;
Vector128<float> y = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref yLaneRef, i)) * scaleVector;
Vector128<float> z = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref zLaneRef, i)) * scaleVector;
Vector128<float> shiftedX = Vector128_.ShiftRightBytesInVector(x.AsByte(), sizeof(float)).AsSingle();
Transpose4(x, y, z, shiftedX, out Vector128<float> pixel0, out Vector128<float> pixel1, out Vector128<float> pixel2, out Vector128<float> pixel3);
// Dropping pixel2.X lets [Y2] complete [Y1 Z1 X2] from pixel1.
Vector128<byte> shiftedPixel2 = Vector128_.ShiftRightBytesInVector(pixel2.AsByte(), sizeof(float));
Vector128<float> packed1 = Vector128_.AlignRight(shiftedPixel2, pixel1.AsByte(), sizeof(float)).AsSingle();
// Dropping pixel3.X leaves [Y3 Z3] to complete [Z2 X3] from pixel2.
Vector128<byte> shiftedPixel3 = Vector128_.ShiftRightBytesInVector(pixel3.AsByte(), sizeof(float));
Vector128<float> packed2 = Vector128_.AlignRight(shiftedPixel3, pixel2.AsByte(), sizeof(float) * 2).AsSingle();
ref Vector128<float> destination = ref Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref packedRef, (uint)i * 3));
destination = pixel0;
Unsafe.Add(ref destination, 1) = packed1;
Unsafe.Add(ref destination, 2) = packed2;
}
}
// Fewer than four pixels remain after SIMD, or every pixel reaches this path
// when the runtime cannot accelerate the cross-vector transpose.
for (; i < xLane.Length; i++)
{
nuint sourceOffset = (uint)i;
nuint packedOffset = sourceOffset * 3;
Unsafe.Add(ref packedRef, packedOffset) = Unsafe.Add(ref xLaneRef, sourceOffset) * scale;
Unsafe.Add(ref packedRef, packedOffset + 1) = Unsafe.Add(ref yLaneRef, sourceOffset) * scale;
Unsafe.Add(ref packedRef, packedOffset + 2) = Unsafe.Add(ref zLaneRef, sourceOffset) * scale;
}
}
/// <summary>
/// Deinterleaves packed XYZ values into three planar component lanes.
/// </summary>
/// <param name="packed">The source ordered as consecutive XYZ triples.</param>
/// <param name="xLane">The destination X components.</param>
/// <param name="yLane">The destination Y components.</param>
/// <param name="zLane">The destination Z components.</param>
public static void UnpackDeinterleave3(ReadOnlySpan<Vector3> packed, Span<float> xLane, Span<float> yLane, Span<float> zLane)
{
DebugGuard.IsTrue(packed.Length == xLane.Length, nameof(packed), "Channels must be of same size!");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
ref float packedRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<Vector3, float>(packed));
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
int i = 0;
if (Vector128.IsHardwareAccelerated)
{
int oneVectorFromEnd = packed.Length - Vector128<float>.Count;
for (; i <= oneVectorFromEnd; i += Vector128<float>.Count)
{
// A Vector3 occupies twelve contiguous bytes, so a sixteen-byte load beginning
// at one pixel also reads the X component of the following pixel:
// pixel0 = [X0 Y0 Z0 X1]
// pixel1 = [X1 Y1 Z1 X2]
// The transpose discards this fourth column, making the overlap useful padding
// and avoiding two insert instructions per pixel. The final row needs explicit
// zero padding only when pixel3 is the last element in the source span.
nuint packedOffset = (uint)i * 3;
Vector128<float> pixel0 = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref packedRef, packedOffset));
Vector128<float> pixel1 = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref packedRef, packedOffset + 3));
Vector128<float> pixel2 = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref packedRef, packedOffset + 6));
ref float pixel3Ref = ref Unsafe.Add(ref packedRef, packedOffset + 9);
Vector128<float> pixel3 = i + Vector128<float>.Count < packed.Length ? Unsafe.As<float, Vector128<float>>(ref pixel3Ref) : Unsafe.As<float, Vector3>(ref pixel3Ref).AsVector128();
Transpose4(pixel0, pixel1, pixel2, pixel3, out Vector128<float> x, out Vector128<float> y, out Vector128<float> z, out _);
Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref xLaneRef, i)) = x;
Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref yLaneRef, i)) = y;
Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref zLaneRef, i)) = z;
}
}
// The scalar remainder preserves the original scatter behavior for zero to
// three pixels and provides the complete fallback on unsupported hardware.
for (; i < packed.Length; i++)
{
nuint packedOffset = (uint)i * 3;
Unsafe.Add(ref xLaneRef, i) = Unsafe.Add(ref packedRef, packedOffset);
Unsafe.Add(ref yLaneRef, i) = Unsafe.Add(ref packedRef, packedOffset + 1);
Unsafe.Add(ref zLaneRef, i) = Unsafe.Add(ref packedRef, packedOffset + 2);
}
}
/// <summary>
/// Normalizes four planar component lanes and interleaves them into packed XYZW values.
/// </summary>
/// <param name="xLane">The planar X components.</param>
/// <param name="yLane">The planar Y components.</param>
/// <param name="zLane">The planar Z components.</param>
/// <param name="wLane">The planar W components.</param>
/// <param name="packed">The destination ordered as consecutive XYZW groups.</param>
/// <param name="maxValue">The maximum component value used to normalize each component.</param>
public static void PackedNormalizeInterleave4(ReadOnlySpan<float> xLane, ReadOnlySpan<float> yLane, ReadOnlySpan<float> zLane, ReadOnlySpan<float> wLane, Span<float> packed, float maxValue)
{
DebugGuard.IsTrue(packed.Length % 4 == 0, "Packed length must be divisible by 4.");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
DebugGuard.IsTrue(wLane.Length == xLane.Length, nameof(wLane), "Channels must be of same size!");
DebugGuard.MustBeLessThanOrEqualTo(packed.Length / 4, xLane.Length, nameof(packed));
float scale = 1F / maxValue;
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float wLaneRef = ref MemoryMarshal.GetReference(wLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
int i = 0;
if (Vector128.IsHardwareAccelerated)
{
Vector128<float> scaleVector = Vector128.Create(scale);
int oneVectorFromEnd = xLane.Length - Vector128<float>.Count;
for (; i <= oneVectorFromEnd; i += Vector128<float>.Count)
{
// Four planar vectors form the rows of a 4x4 matrix. Transposition
// converts them into four complete [Xn Yn Zn Wn] pixel vectors, so
// normalization and interleaving require only four loads, four
// multiplies, the register transpose, and four contiguous stores.
Vector128<float> x = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref xLaneRef, i)) * scaleVector;
Vector128<float> y = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref yLaneRef, i)) * scaleVector;
Vector128<float> z = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref zLaneRef, i)) * scaleVector;
Vector128<float> w = Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref wLaneRef, i)) * scaleVector;
Transpose4(x, y, z, w, out Vector128<float> pixel0, out Vector128<float> pixel1, out Vector128<float> pixel2, out Vector128<float> pixel3);
ref Vector128<float> destination = ref Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref packedRef, (uint)i * 4));
destination = pixel0;
Unsafe.Add(ref destination, 1) = pixel1;
Unsafe.Add(ref destination, 2) = pixel2;
Unsafe.Add(ref destination, 3) = pixel3;
}
}
// Process the zero-to-three trailing pixels with the same normalization
// and component order as the vector transpose.
for (; i < xLane.Length; i++)
{
nuint sourceOffset = (uint)i;
nuint packedOffset = sourceOffset * 4;
Unsafe.Add(ref packedRef, packedOffset) = Unsafe.Add(ref xLaneRef, sourceOffset) * scale;
Unsafe.Add(ref packedRef, packedOffset + 1) = Unsafe.Add(ref yLaneRef, sourceOffset) * scale;
Unsafe.Add(ref packedRef, packedOffset + 2) = Unsafe.Add(ref zLaneRef, sourceOffset) * scale;
Unsafe.Add(ref packedRef, packedOffset + 3) = Unsafe.Add(ref wLaneRef, sourceOffset) * scale;
}
}
/// <summary>
/// Inverts and normalizes four planar component lanes before interleaving them into packed XYZW values.
/// </summary>
/// <param name="xLane">The inverted planar X components.</param>
/// <param name="yLane">The inverted planar Y components.</param>
/// <param name="zLane">The inverted planar Z components.</param>
/// <param name="wLane">The inverted planar W components.</param>
/// <param name="packed">The destination ordered as consecutive conventional XYZW groups.</param>
/// <param name="maxValue">The maximum component value used for inversion and normalization.</param>
public static void PackedInvertNormalizeInterleave4(ReadOnlySpan<float> xLane, ReadOnlySpan<float> yLane, ReadOnlySpan<float> zLane, ReadOnlySpan<float> wLane, Span<float> packed, float maxValue)
{
DebugGuard.IsTrue(packed.Length % 4 == 0, "Packed length must be divisible by 4.");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
DebugGuard.IsTrue(wLane.Length == xLane.Length, nameof(wLane), "Channels must be of same size!");
DebugGuard.MustBeLessThanOrEqualTo(packed.Length / 4, xLane.Length, nameof(packed));
float scale = 1F / maxValue;
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float wLaneRef = ref MemoryMarshal.GetReference(wLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
int i = 0;
if (Vector128.IsHardwareAccelerated)
{
Vector128<float> maximumVector = Vector128.Create(maxValue);
Vector128<float> scaleVector = Vector128.Create(scale);
int oneVectorFromEnd = xLane.Length - Vector128<float>.Count;
for (; i <= oneVectorFromEnd; i += Vector128<float>.Count)
{
// Adobe JPEG stores all four components inverted in the sample
// domain. Reflecting and normalizing the planar vectors before the
// transpose keeps both arithmetic operations lane-wise and leaves
// the transpose responsible only for the planar-to-packed layout.
Vector128<float> x = (maximumVector - Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref xLaneRef, i))) * scaleVector;
Vector128<float> y = (maximumVector - Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref yLaneRef, i))) * scaleVector;
Vector128<float> z = (maximumVector - Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref zLaneRef, i))) * scaleVector;
Vector128<float> w = (maximumVector - Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref wLaneRef, i))) * scaleVector;
Transpose4(x, y, z, w, out Vector128<float> pixel0, out Vector128<float> pixel1, out Vector128<float> pixel2, out Vector128<float> pixel3);
ref Vector128<float> destination = ref Unsafe.As<float, Vector128<float>>(ref Unsafe.Add(ref packedRef, (uint)i * 4));
destination = pixel0;
Unsafe.Add(ref destination, 1) = pixel1;
Unsafe.Add(ref destination, 2) = pixel2;
Unsafe.Add(ref destination, 3) = pixel3;
}
}
// Preserve the original operation order for the zero-to-three trailing pixels:
// subtract in the sample domain, then multiply by the reciprocal maximum.
for (; i < xLane.Length; i++)
{
nuint sourceOffset = (uint)i;
nuint packedOffset = sourceOffset * 4;
Unsafe.Add(ref packedRef, packedOffset) = (maxValue - Unsafe.Add(ref xLaneRef, sourceOffset)) * scale;
Unsafe.Add(ref packedRef, packedOffset + 1) = (maxValue - Unsafe.Add(ref yLaneRef, sourceOffset)) * scale;
Unsafe.Add(ref packedRef, packedOffset + 2) = (maxValue - Unsafe.Add(ref zLaneRef, sourceOffset)) * scale;
Unsafe.Add(ref packedRef, packedOffset + 3) = (maxValue - Unsafe.Add(ref wLaneRef, sourceOffset)) * scale;
}
}
/// <summary>
/// Transposes four four-lane rows into four four-lane columns.
/// </summary>
/// <param name="row0">The first matrix row.</param>
/// <param name="row1">The second matrix row.</param>
/// <param name="row2">The third matrix row.</param>
/// <param name="row3">The fourth matrix row.</param>
/// <param name="column0">The first matrix column.</param>
/// <param name="column1">The second matrix column.</param>
/// <param name="column2">The third matrix column.</param>
/// <param name="column3">The fourth matrix column.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void Transpose4(Vector128<float> row0, Vector128<float> row1, Vector128<float> row2, Vector128<float> row3, out Vector128<float> column0, out Vector128<float> column1, out Vector128<float> column2, out Vector128<float> column3)
{
// The first unpack interleaves adjacent 32-bit lanes from rows 0/1 and 2/3:
// row01Low = [r0c0 r1c0 r0c1 r1c1]
// row23Low = [r2c0 r3c0 r2c1 r3c1]
// A second unpack treats each adjacent pair as one 64-bit lane and combines
// the row01 and row23 pairs into complete columns. The integer views only
// expose the cross-platform unpack helpers; every floating-point bit is preserved.
Vector128<int> row01Low = Vector128_.UnpackLow(row0.AsInt32(), row1.AsInt32());
Vector128<int> row01High = Vector128_.UnpackHigh(row0.AsInt32(), row1.AsInt32());
Vector128<int> row23Low = Vector128_.UnpackLow(row2.AsInt32(), row3.AsInt32());
Vector128<int> row23High = Vector128_.UnpackHigh(row2.AsInt32(), row3.AsInt32());
column0 = Vector128_.UnpackLow(row01Low.AsInt64(), row23Low.AsInt64()).AsSingle();
column1 = Vector128_.UnpackHigh(row01Low.AsInt64(), row23Low.AsInt64()).AsSingle();
column2 = Vector128_.UnpackLow(row01High.AsInt64(), row23High.AsInt64()).AsSingle();
column3 = Vector128_.UnpackHigh(row01High.AsInt64(), row23High.AsInt64()).AsSingle();
}
}

90
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.RgbOperator.cs

@ -27,14 +27,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref float c0, ref float c1, ref float c2, float c3, float maximumValue, float halfValue, float scale)
ref float c0,
ref float c1,
ref float c2,
float c3,
float maximumValue,
float halfValue,
float scale)
{ {
// The JPEG planes already represent R, G, and B. Conversion therefore consists only of moving // The JPEG planes already represent R, G, and B. Conversion therefore consists only of moving
// each integer-domain sample into the normalized floating-point domain consumed by pixel packing. // each integer-domain sample into the normalized floating-point domain consumed by pixel packing.
@ -45,14 +38,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector128<float> c0, ref Vector128<float> c1, ref Vector128<float> c2, Vector128<float> c3, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale)
ref Vector128<float> c0,
ref Vector128<float> c1,
ref Vector128<float> c2,
Vector128<float> c3,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale)
{ {
// Four samples from each planar channel remain in their lanes while sharing one normalization vector. // Four samples from each planar channel remain in their lanes while sharing one normalization vector.
c0 *= scale; c0 *= scale;
@ -62,14 +48,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector256<float> c0, ref Vector256<float> c1, ref Vector256<float> c2, Vector256<float> c3, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale)
ref Vector256<float> c0,
ref Vector256<float> c1,
ref Vector256<float> c2,
Vector256<float> c3,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale)
{ {
// Eight samples per plane are normalized independently without channel shuffles. // Eight samples per plane are normalized independently without channel shuffles.
c0 *= scale; c0 *= scale;
@ -79,14 +58,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector512<float> c0, ref Vector512<float> c1, ref Vector512<float> c2, Vector512<float> c3, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale)
ref Vector512<float> c0,
ref Vector512<float> c1,
ref Vector512<float> c2,
Vector512<float> c3,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale)
{ {
// Sixteen samples per plane are normalized independently without changing planar ordering. // Sixteen samples per plane are normalized independently without changing planar ordering.
c0 *= scale; c0 *= scale;
@ -96,17 +68,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(float r, float g, float b, float maximumValue, float halfValue, float scale, out float c0, out float c1, out float c2, out float c3)
float r,
float g,
float b,
float maximumValue,
float halfValue,
float scale,
out float c0,
out float c1,
out float c2,
out float c3)
{ {
// Encoder RGB lanes already use the JPEG sample domain, so the direct color model copies them. // Encoder RGB lanes already use the JPEG sample domain, so the direct color model copies them.
c0 = r; c0 = r;
@ -117,17 +79,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector128<float> r, Vector128<float> g, Vector128<float> b, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale, out Vector128<float> c0, out Vector128<float> c1, out Vector128<float> c2, out Vector128<float> c3)
Vector128<float> r,
Vector128<float> g,
Vector128<float> b,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3)
{ {
// The planar vectors map one-to-one to JPEG components; the fourth result is statically discarded. // The planar vectors map one-to-one to JPEG components; the fourth result is statically discarded.
c0 = r; c0 = r;
@ -138,17 +90,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector256<float> r, Vector256<float> g, Vector256<float> b, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale, out Vector256<float> c0, out Vector256<float> c1, out Vector256<float> c2, out Vector256<float> c3)
Vector256<float> r,
Vector256<float> g,
Vector256<float> b,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3)
{ {
// The planar vectors map one-to-one to JPEG components; no arithmetic or rearrangement is required. // The planar vectors map one-to-one to JPEG components; no arithmetic or rearrangement is required.
c0 = r; c0 = r;
@ -159,17 +101,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector512<float> r, Vector512<float> g, Vector512<float> b, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale, out Vector512<float> c0, out Vector512<float> c1, out Vector512<float> c2, out Vector512<float> c3)
Vector512<float> r,
Vector512<float> g,
Vector512<float> b,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3)
{ {
// The widest path is likewise a register-to-register planar copy for sixteen pixels. // The widest path is likewise a register-to-register planar copy for sixteen pixels.
c0 = r; c0 = r;
@ -179,11 +111,7 @@ internal abstract partial class JpegColorConverterBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public static void ConvertToRgbInPlaceWithIcc( public static void ConvertToRgbInPlaceWithIcc(Configuration configuration, IccProfile profile, in ComponentValues values, float maximumValue)
Configuration configuration,
IccProfile profile,
in ComponentValues values,
float maximumValue)
{ {
using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 3); using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 3);
Span<float> packed = memoryOwner.Memory.Span; Span<float> packed = memoryOwner.Memory.Span;

150
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YCbCrOperator.cs

@ -48,14 +48,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref float c0, ref float c1, ref float c2, float c3, float maximumValue, float halfValue, float scale)
ref float c0,
ref float c1,
ref float c2,
float c3,
float maximumValue,
float halfValue,
float scale)
{ {
float y = c0; float y = c0;
float cb = c1 - halfValue; float cb = c1 - halfValue;
@ -72,14 +65,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector128<float> c0, ref Vector128<float> c1, ref Vector128<float> c2, Vector128<float> c3, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale)
ref Vector128<float> c0,
ref Vector128<float> c1,
ref Vector128<float> c2,
Vector128<float> c3,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale)
{ {
Vector128<float> y = c0; Vector128<float> y = c0;
Vector128<float> cb = c1 - halfValue; Vector128<float> cb = c1 - halfValue;
@ -89,10 +75,7 @@ internal abstract partial class JpegColorConverterBase
// R uses Cr, B uses Cb, and G subtracts both chroma contributions. Rounding occurs in the sample // R uses Cr, B uses Cb, and G subtracts both chroma contributions. Rounding occurs in the sample
// domain before the common normalization scale so all precisions use integer JPEG sample semantics. // domain before the common normalization scale so all precisions use integer JPEG sample semantics.
Vector128<float> r = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(RCrMult), y); Vector128<float> r = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(RCrMult), y);
Vector128<float> g = Vector128_.MultiplyAddEstimate( Vector128<float> g = Vector128_.MultiplyAddEstimate(cr, Vector128.Create(-GCrMult), Vector128_.MultiplyAddEstimate(cb, Vector128.Create(-GCbMult), y));
cr,
Vector128.Create(-GCrMult),
Vector128_.MultiplyAddEstimate(cb, Vector128.Create(-GCbMult), y));
Vector128<float> b = Vector128_.MultiplyAddEstimate(cb, Vector128.Create(BCbMult), y); Vector128<float> b = Vector128_.MultiplyAddEstimate(cb, Vector128.Create(BCbMult), y);
c0 = Vector128_.RoundToNearestInteger(r) * scale; c0 = Vector128_.RoundToNearestInteger(r) * scale;
@ -102,14 +85,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector256<float> c0, ref Vector256<float> c1, ref Vector256<float> c2, Vector256<float> c3, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale)
ref Vector256<float> c0,
ref Vector256<float> c1,
ref Vector256<float> c2,
Vector256<float> c3,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale)
{ {
Vector256<float> y = c0; Vector256<float> y = c0;
Vector256<float> cb = c1 - halfValue; Vector256<float> cb = c1 - halfValue;
@ -119,10 +95,7 @@ internal abstract partial class JpegColorConverterBase
// Keeping an explicit overload allows the JIT to emit native YMM operations without a width // Keeping an explicit overload allows the JIT to emit native YMM operations without a width
// switch or decomposing the vector into smaller values. // switch or decomposing the vector into smaller values.
Vector256<float> r = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(RCrMult), y); Vector256<float> r = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(RCrMult), y);
Vector256<float> g = Vector256_.MultiplyAddEstimate( Vector256<float> g = Vector256_.MultiplyAddEstimate(cr, Vector256.Create(-GCrMult), Vector256_.MultiplyAddEstimate(cb, Vector256.Create(-GCbMult), y));
cr,
Vector256.Create(-GCrMult),
Vector256_.MultiplyAddEstimate(cb, Vector256.Create(-GCbMult), y));
Vector256<float> b = Vector256_.MultiplyAddEstimate(cb, Vector256.Create(BCbMult), y); Vector256<float> b = Vector256_.MultiplyAddEstimate(cb, Vector256.Create(BCbMult), y);
c0 = Vector256_.RoundToNearestInteger(r) * scale; c0 = Vector256_.RoundToNearestInteger(r) * scale;
@ -132,14 +105,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertToRgb( public static void ConvertToRgb(ref Vector512<float> c0, ref Vector512<float> c1, ref Vector512<float> c2, Vector512<float> c3, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale)
ref Vector512<float> c0,
ref Vector512<float> c1,
ref Vector512<float> c2,
Vector512<float> c3,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale)
{ {
Vector512<float> y = c0; Vector512<float> y = c0;
Vector512<float> cb = c1 - halfValue; Vector512<float> cb = c1 - halfValue;
@ -149,10 +115,7 @@ internal abstract partial class JpegColorConverterBase
// assembly inspection verifies the JIT hoists them from the loop and retains fused operations. // assembly inspection verifies the JIT hoists them from the loop and retains fused operations.
// The formula and rounding order remain identical to the narrower overloads. // The formula and rounding order remain identical to the narrower overloads.
Vector512<float> r = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(RCrMult), y); Vector512<float> r = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(RCrMult), y);
Vector512<float> g = Vector512_.MultiplyAddEstimate( Vector512<float> g = Vector512_.MultiplyAddEstimate(cr, Vector512.Create(-GCrMult), Vector512_.MultiplyAddEstimate(cb, Vector512.Create(-GCbMult), y));
cr,
Vector512.Create(-GCrMult),
Vector512_.MultiplyAddEstimate(cb, Vector512.Create(-GCbMult), y));
Vector512<float> b = Vector512_.MultiplyAddEstimate(cb, Vector512.Create(BCbMult), y); Vector512<float> b = Vector512_.MultiplyAddEstimate(cb, Vector512.Create(BCbMult), y);
c0 = Vector512_.RoundToNearestInteger(r) * scale; c0 = Vector512_.RoundToNearestInteger(r) * scale;
@ -162,17 +125,7 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(float r, float g, float b, float maximumValue, float halfValue, float scale, out float c0, out float c1, out float c2, out float c3)
float r,
float g,
float b,
float maximumValue,
float halfValue,
float scale,
out float c0,
out float c1,
out float c2,
out float c3)
{ {
// The RGB inputs are unnormalized 0..255 encoder lanes. The BT.601 luma weights form Y, // The RGB inputs are unnormalized 0..255 encoder lanes. The BT.601 luma weights form Y,
// while the signed chroma projections are biased by halfValue into the JPEG sample domain. // while the signed chroma projections are biased by halfValue into the JPEG sample domain.
@ -185,104 +138,43 @@ internal abstract partial class JpegColorConverterBase
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector128<float> r, Vector128<float> g, Vector128<float> b, Vector128<float> maximumValue, Vector128<float> halfValue, Vector128<float> scale, out Vector128<float> c0, out Vector128<float> c1, out Vector128<float> c2, out Vector128<float> c3)
Vector128<float> r,
Vector128<float> g,
Vector128<float> b,
Vector128<float> maximumValue,
Vector128<float> halfValue,
Vector128<float> scale,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3)
{ {
// Each vector holds four consecutive values from one RGB plane. The nested multiply-add sequence // Each vector holds four consecutive values from one RGB plane. The nested multiply-add sequence
// produces four Y lanes, four Cb lanes, and four Cr lanes without transposition. The association // produces four Y lanes, four Cb lanes, and four Cr lanes without transposition. The association
// exposes two FMA opportunities per output while preserving the scalar formula's term grouping. // exposes two FMA opportunities per output while preserving the scalar formula's term grouping.
c0 = Vector128_.MultiplyAddEstimate( c0 = Vector128_.MultiplyAddEstimate(Vector128.Create(0.299F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b));
Vector128.Create(0.299F), c1 = halfValue + Vector128_.MultiplyAddEstimate(Vector128.Create(-0.168736F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(-0.331264F), g, Vector128.Create(0.5F) * b));
r, c2 = halfValue + Vector128_.MultiplyAddEstimate(Vector128.Create(0.5F), r, Vector128_.MultiplyAddEstimate(Vector128.Create(-0.418688F), g, Vector128.Create(-0.081312F) * b));
Vector128_.MultiplyAddEstimate(Vector128.Create(0.587F), g, Vector128.Create(0.114F) * b));
c1 = halfValue + Vector128_.MultiplyAddEstimate(
Vector128.Create(-0.168736F),
r,
Vector128_.MultiplyAddEstimate(Vector128.Create(-0.331264F), g, Vector128.Create(0.5F) * b));
c2 = halfValue + Vector128_.MultiplyAddEstimate(
Vector128.Create(0.5F),
r,
Vector128_.MultiplyAddEstimate(Vector128.Create(-0.418688F), g, Vector128.Create(-0.081312F) * b));
c3 = default; c3 = default;
} }
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector256<float> r, Vector256<float> g, Vector256<float> b, Vector256<float> maximumValue, Vector256<float> halfValue, Vector256<float> scale, out Vector256<float> c0, out Vector256<float> c1, out Vector256<float> c2, out Vector256<float> c3)
Vector256<float> r,
Vector256<float> g,
Vector256<float> b,
Vector256<float> maximumValue,
Vector256<float> halfValue,
Vector256<float> scale,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3)
{ {
// Eight planar RGB samples use the identical association as Vector128, allowing direct YMM FMA // Eight planar RGB samples use the identical association as Vector128, allowing direct YMM FMA
// generation while preserving the component-per-vector output layout. // generation while preserving the component-per-vector output layout.
c0 = Vector256_.MultiplyAddEstimate( c0 = Vector256_.MultiplyAddEstimate(Vector256.Create(0.299F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b));
Vector256.Create(0.299F), c1 = halfValue + Vector256_.MultiplyAddEstimate(Vector256.Create(-0.168736F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(-0.331264F), g, Vector256.Create(0.5F) * b));
r, c2 = halfValue + Vector256_.MultiplyAddEstimate(Vector256.Create(0.5F), r, Vector256_.MultiplyAddEstimate(Vector256.Create(-0.418688F), g, Vector256.Create(-0.081312F) * b));
Vector256_.MultiplyAddEstimate(Vector256.Create(0.587F), g, Vector256.Create(0.114F) * b));
c1 = halfValue + Vector256_.MultiplyAddEstimate(
Vector256.Create(-0.168736F),
r,
Vector256_.MultiplyAddEstimate(Vector256.Create(-0.331264F), g, Vector256.Create(0.5F) * b));
c2 = halfValue + Vector256_.MultiplyAddEstimate(
Vector256.Create(0.5F),
r,
Vector256_.MultiplyAddEstimate(Vector256.Create(-0.418688F), g, Vector256.Create(-0.081312F) * b));
c3 = default; c3 = default;
} }
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ConvertFromRgb( public static void ConvertFromRgb(Vector512<float> r, Vector512<float> g, Vector512<float> b, Vector512<float> maximumValue, Vector512<float> halfValue, Vector512<float> scale, out Vector512<float> c0, out Vector512<float> c1, out Vector512<float> c2, out Vector512<float> c3)
Vector512<float> r,
Vector512<float> g,
Vector512<float> b,
Vector512<float> maximumValue,
Vector512<float> halfValue,
Vector512<float> scale,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3)
{ {
// Sixteen planar RGB samples use the same nested form. Constants are lane broadcasts and c3 is // Sixteen planar RGB samples use the same nested form. Constants are lane broadcasts and c3 is
// deliberately zero because the shared traversal removes the unused fourth store for this operator. // deliberately zero because the shared traversal removes the unused fourth store for this operator.
c0 = Vector512_.MultiplyAddEstimate( c0 = Vector512_.MultiplyAddEstimate(Vector512.Create(0.299F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b));
Vector512.Create(0.299F), c1 = halfValue + Vector512_.MultiplyAddEstimate(Vector512.Create(-0.168736F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(-0.331264F), g, Vector512.Create(0.5F) * b));
r, c2 = halfValue + Vector512_.MultiplyAddEstimate(Vector512.Create(0.5F), r, Vector512_.MultiplyAddEstimate(Vector512.Create(-0.418688F), g, Vector512.Create(-0.081312F) * b));
Vector512_.MultiplyAddEstimate(Vector512.Create(0.587F), g, Vector512.Create(0.114F) * b));
c1 = halfValue + Vector512_.MultiplyAddEstimate(
Vector512.Create(-0.168736F),
r,
Vector512_.MultiplyAddEstimate(Vector512.Create(-0.331264F), g, Vector512.Create(0.5F) * b));
c2 = halfValue + Vector512_.MultiplyAddEstimate(
Vector512.Create(0.5F),
r,
Vector512_.MultiplyAddEstimate(Vector512.Create(-0.418688F), g, Vector512.Create(-0.081312F) * b));
c3 = default; c3 = default;
} }
/// <inheritdoc/> /// <inheritdoc/>
public static void ConvertToRgbInPlaceWithIcc( public static void ConvertToRgbInPlaceWithIcc(Configuration configuration, IccProfile profile, in ComponentValues values, float maximumValue)
Configuration configuration,
IccProfile profile,
in ComponentValues values,
float maximumValue)
{ {
using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 3); using IMemoryOwner<float> memoryOwner = configuration.MemoryAllocator.Allocate<float>(values.Component0.Length * 3);
Span<float> packed = memoryOwner.Memory.Span; Span<float> packed = memoryOwner.Memory.Span;

4
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.YccKOperator.cs

@ -103,6 +103,7 @@ internal abstract partial class JpegColorConverterBase
// CMYK extraction supplies inverted chromatic samples and K. Reflecting the first three results // CMYK extraction supplies inverted chromatic samples and K. Reflecting the first three results
// reconstructs the chromatic RGB that YCbCr encodes, while K passes through untouched. // reconstructs the chromatic RGB that YCbCr encodes, while K passes through untouched.
CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out float c, out float m, out float y, out c3); CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out float c, out float m, out float y, out c3);
YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _); YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _);
} }
@ -112,6 +113,7 @@ internal abstract partial class JpegColorConverterBase
{ {
// Static constrained calls inline both stages, keeping four pixels in registers without materializing CMYK planes. // Static constrained calls inline both stages, keeping four pixels in registers without materializing CMYK planes.
CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out Vector128<float> c, out Vector128<float> m, out Vector128<float> y, out c3); CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out Vector128<float> c, out Vector128<float> m, out Vector128<float> y, out c3);
YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _); YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _);
} }
@ -121,6 +123,7 @@ internal abstract partial class JpegColorConverterBase
{ {
// Eight pixels flow through CMYK extraction and YCbCr projection entirely in YMM registers. // Eight pixels flow through CMYK extraction and YCbCr projection entirely in YMM registers.
CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out Vector256<float> c, out Vector256<float> m, out Vector256<float> y, out c3); CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out Vector256<float> c, out Vector256<float> m, out Vector256<float> y, out c3);
YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _); YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _);
} }
@ -130,6 +133,7 @@ internal abstract partial class JpegColorConverterBase
{ {
// Sixteen pixels flow through both mathematical stages in registers without materializing intermediate planes. // Sixteen pixels flow through both mathematical stages in registers without materializing intermediate planes.
CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out Vector512<float> c, out Vector512<float> m, out Vector512<float> y, out c3); CmykOperator.ConvertFromRgb(r, g, b, maximumValue, halfValue, scale, out Vector512<float> c, out Vector512<float> m, out Vector512<float> y, out c3);
YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _); YCbCrOperator.ConvertFromRgb(maximumValue - c, maximumValue - m, maximumValue - y, maximumValue, halfValue, scale, out c0, out c1, out c2, out _);
} }

135
src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverterBase.cs

@ -2,9 +2,6 @@
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
#nullable disable #nullable disable
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.Metadata.Profiles.Icc; using SixLabors.ImageSharp.Metadata.Profiles.Icc;
@ -101,124 +98,6 @@ internal abstract partial class JpegColorConverterBase
/// <param name="bLane">Blue colors lane.</param> /// <param name="bLane">Blue colors lane.</param>
public abstract void ConvertFromRgb(in ComponentValues values, Span<float> rLane, Span<float> gLane, Span<float> bLane); public abstract void ConvertFromRgb(in ComponentValues values, Span<float> rLane, Span<float> gLane, Span<float> bLane);
public static void PackedNormalizeInterleave3(
ReadOnlySpan<float> xLane,
ReadOnlySpan<float> yLane,
ReadOnlySpan<float> zLane,
Span<float> packed,
float scale)
{
DebugGuard.IsTrue(packed.Length % 3 == 0, "Packed length must be divisible by 3.");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
DebugGuard.MustBeLessThanOrEqualTo(packed.Length / 3, xLane.Length, nameof(packed));
// TODO: Investigate SIMD version of this.
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
for (nuint i = 0; i < (nuint)xLane.Length; i++)
{
nuint baseIdx = i * 3;
Unsafe.Add(ref packedRef, baseIdx) = Unsafe.Add(ref xLaneRef, i) * scale;
Unsafe.Add(ref packedRef, baseIdx + 1) = Unsafe.Add(ref yLaneRef, i) * scale;
Unsafe.Add(ref packedRef, baseIdx + 2) = Unsafe.Add(ref zLaneRef, i) * scale;
}
}
public static void UnpackDeinterleave3(
ReadOnlySpan<Vector3> packed,
Span<float> xLane,
Span<float> yLane,
Span<float> zLane)
{
DebugGuard.IsTrue(packed.Length == xLane.Length, nameof(packed), "Channels must be of same size!");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
// TODO: Investigate SIMD version of this.
ref float packedRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<Vector3, float>(packed));
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
for (nuint i = 0; i < (nuint)packed.Length; i++)
{
nuint baseIdx = i * 3;
Unsafe.Add(ref xLaneRef, i) = Unsafe.Add(ref packedRef, baseIdx);
Unsafe.Add(ref yLaneRef, i) = Unsafe.Add(ref packedRef, baseIdx + 1);
Unsafe.Add(ref zLaneRef, i) = Unsafe.Add(ref packedRef, baseIdx + 2);
}
}
public static void PackedNormalizeInterleave4(
ReadOnlySpan<float> xLane,
ReadOnlySpan<float> yLane,
ReadOnlySpan<float> zLane,
ReadOnlySpan<float> wLane,
Span<float> packed,
float maxValue)
{
DebugGuard.IsTrue(packed.Length % 4 == 0, "Packed length must be divisible by 4.");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
DebugGuard.IsTrue(wLane.Length == xLane.Length, nameof(wLane), "Channels must be of same size!");
DebugGuard.MustBeLessThanOrEqualTo(packed.Length / 4, xLane.Length, nameof(packed));
float scale = 1F / maxValue;
// TODO: Investigate SIMD version of this.
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float wLaneRef = ref MemoryMarshal.GetReference(wLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
for (nuint i = 0; i < (nuint)xLane.Length; i++)
{
nuint baseIdx = i * 4;
Unsafe.Add(ref packedRef, baseIdx) = Unsafe.Add(ref xLaneRef, i) * scale;
Unsafe.Add(ref packedRef, baseIdx + 1) = Unsafe.Add(ref yLaneRef, i) * scale;
Unsafe.Add(ref packedRef, baseIdx + 2) = Unsafe.Add(ref zLaneRef, i) * scale;
Unsafe.Add(ref packedRef, baseIdx + 3) = Unsafe.Add(ref wLaneRef, i) * scale;
}
}
public static void PackedInvertNormalizeInterleave4(
ReadOnlySpan<float> xLane,
ReadOnlySpan<float> yLane,
ReadOnlySpan<float> zLane,
ReadOnlySpan<float> wLane,
Span<float> packed,
float maxValue)
{
DebugGuard.IsTrue(packed.Length % 4 == 0, "Packed length must be divisible by 4.");
DebugGuard.IsTrue(yLane.Length == xLane.Length, nameof(yLane), "Channels must be of same size!");
DebugGuard.IsTrue(zLane.Length == xLane.Length, nameof(zLane), "Channels must be of same size!");
DebugGuard.IsTrue(wLane.Length == xLane.Length, nameof(wLane), "Channels must be of same size!");
DebugGuard.MustBeLessThanOrEqualTo(packed.Length / 4, xLane.Length, nameof(packed));
float scale = 1F / maxValue;
// TODO: Investigate SIMD version of this.
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float wLaneRef = ref MemoryMarshal.GetReference(wLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
for (nuint i = 0; i < (nuint)xLane.Length; i++)
{
nuint baseIdx = i * 4;
Unsafe.Add(ref packedRef, baseIdx) = (maxValue - Unsafe.Add(ref xLaneRef, i)) * scale;
Unsafe.Add(ref packedRef, baseIdx + 1) = (maxValue - Unsafe.Add(ref yLaneRef, i)) * scale;
Unsafe.Add(ref packedRef, baseIdx + 2) = (maxValue - Unsafe.Add(ref zLaneRef, i)) * scale;
Unsafe.Add(ref packedRef, baseIdx + 3) = (maxValue - Unsafe.Add(ref wLaneRef, i)) * scale;
}
}
/// <summary> /// <summary>
/// Returns the <see cref="JpegColorConverterBase"/>s for all supported color spaces and precisions. /// Returns the <see cref="JpegColorConverterBase"/>s for all supported color spaces and precisions.
/// </summary> /// </summary>
@ -382,6 +261,14 @@ internal abstract partial class JpegColorConverterBase
this.Component3 = this.ComponentCount > 3 ? processors[3].GetColorBufferRowSpan(row) : []; this.Component3 = this.ComponentCount > 3 ? processors[3].GetColorBufferRowSpan(row) : [];
} }
/// <summary>
/// Initializes a new instance of the <see cref="ComponentValues"/> struct from explicitly supplied planar spans.
/// </summary>
/// <param name="componentCount">The number of populated component planes.</param>
/// <param name="c0">The first component plane.</param>
/// <param name="c1">The second component plane, if present.</param>
/// <param name="c2">The third component plane, if present.</param>
/// <param name="c3">The fourth component plane, if present.</param>
internal ComponentValues( internal ComponentValues(
int componentCount, int componentCount,
Span<float> c0, Span<float> c0,
@ -396,6 +283,12 @@ internal abstract partial class JpegColorConverterBase
this.Component3 = c3; this.Component3 = c3;
} }
/// <summary>
/// Creates a view over the same component planes for the requested sample range.
/// </summary>
/// <param name="start">The zero-based sample offset.</param>
/// <param name="length">The number of samples in each returned plane.</param>
/// <returns>The sliced component values.</returns>
public ComponentValues Slice(int start, int length) public ComponentValues Slice(int start, int length)
{ {
Span<float> c0 = this.Component0.Slice(start, length); Span<float> c0 = this.Component0.Slice(start, length);

4
src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs

@ -116,6 +116,8 @@ internal class ComponentProcessor : IDisposable
} }
static void SumVertical(Span<float> target, Span<float> source) static void SumVertical(Span<float> target, Span<float> source)
// Exact destination overlap is supported, so each accumulated row remains in target.
=> TensorPrimitives_.Add(target, source, target); => TensorPrimitives_.Add(target, source, target);
static void SumHorizontal(Span<float> target, int factor) static void SumHorizontal(Span<float> target, int factor)
@ -164,6 +166,8 @@ internal class ComponentProcessor : IDisposable
} }
static void MultiplyToAverage(Span<float> target, float multiplier) static void MultiplyToAverage(Span<float> target, float multiplier)
// Apply the subsampling reciprocal in place after all contributing rows have been summed.
=> TensorPrimitives_.Multiply(target, multiplier, target); => TensorPrimitives_.Multiply(target, multiplier, target);
} }
} }

7
src/ImageSharp/Formats/Png/Filters/AverageFilter.cs

@ -140,12 +140,7 @@ internal static class AverageFilter
/// <param name="sum">The sum of the total variance of the filtered row.</param> /// <param name="sum">The sum of the total variance of the filtered row.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Encode(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, uint bytesPerPixel, out int sum) public static void Encode(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, uint bytesPerPixel, out int sum)
=> PngFilterEncoder.Encode<AverageFilterOperator>( => PngFilterEncoder.Encode<AverageFilterOperator>(scanline, previousScanline, result, bytesPerPixel, out sum);
scanline,
previousScanline,
result,
bytesPerPixel,
out sum);
/// <summary> /// <summary>
/// Calculates the average value of two bytes /// Calculates the average value of two bytes

341
src/ImageSharp/Formats/Png/Filters/IPngFilterOperator.cs

@ -3,8 +3,8 @@
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics; using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86; using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Formats.Png.Filters; namespace SixLabors.ImageSharp.Formats.Png.Filters;
@ -51,11 +51,7 @@ internal interface IPngFilterOperator
/// <param name="above">The corresponding components in the preceding scanline.</param> /// <param name="above">The corresponding components in the preceding scanline.</param>
/// <param name="upperLeft">The preceding components in the preceding scanline.</param> /// <param name="upperLeft">The preceding components in the preceding scanline.</param>
/// <returns>The filtered residuals.</returns> /// <returns>The filtered residuals.</returns>
public static abstract Vector128<byte> Invoke( public static abstract Vector128<byte> Invoke(Vector128<byte> scan, Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft);
Vector128<byte> scan,
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft);
/// <summary> /// <summary>
/// Filters thirty-two byte lanes from their PNG neighborhoods. /// Filters thirty-two byte lanes from their PNG neighborhoods.
@ -65,11 +61,7 @@ internal interface IPngFilterOperator
/// <param name="above">The corresponding components in the preceding scanline.</param> /// <param name="above">The corresponding components in the preceding scanline.</param>
/// <param name="upperLeft">The preceding components in the preceding scanline.</param> /// <param name="upperLeft">The preceding components in the preceding scanline.</param>
/// <returns>The filtered residuals.</returns> /// <returns>The filtered residuals.</returns>
public static abstract Vector256<byte> Invoke( public static abstract Vector256<byte> Invoke(Vector256<byte> scan, Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft);
Vector256<byte> scan,
Vector256<byte> left,
Vector256<byte> above,
Vector256<byte> upperLeft);
/// <summary> /// <summary>
/// Filters sixty-four byte lanes from their PNG neighborhoods. /// Filters sixty-four byte lanes from their PNG neighborhoods.
@ -79,11 +71,7 @@ internal interface IPngFilterOperator
/// <param name="above">The corresponding components in the preceding scanline.</param> /// <param name="above">The corresponding components in the preceding scanline.</param>
/// <param name="upperLeft">The preceding components in the preceding scanline.</param> /// <param name="upperLeft">The preceding components in the preceding scanline.</param>
/// <returns>The filtered residuals.</returns> /// <returns>The filtered residuals.</returns>
public static abstract Vector512<byte> Invoke( public static abstract Vector512<byte> Invoke(Vector512<byte> scan, Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft);
Vector512<byte> scan,
Vector512<byte> left,
Vector512<byte> above,
Vector512<byte> upperLeft);
} }
/// <summary> /// <summary>
@ -109,29 +97,17 @@ internal readonly struct SubFilterOperator : IPngFilterOperator
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector128<byte> Invoke( public static Vector128<byte> Invoke(Vector128<byte> scan, Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft)
Vector128<byte> scan,
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft)
=> scan - left; => scan - left;
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector256<byte> Invoke( public static Vector256<byte> Invoke(Vector256<byte> scan, Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft)
Vector256<byte> scan,
Vector256<byte> left,
Vector256<byte> above,
Vector256<byte> upperLeft)
=> scan - left; => scan - left;
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector512<byte> Invoke( public static Vector512<byte> Invoke(Vector512<byte> scan, Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft)
Vector512<byte> scan,
Vector512<byte> left,
Vector512<byte> above,
Vector512<byte> upperLeft)
=> scan - left; => scan - left;
} }
@ -158,29 +134,17 @@ internal readonly struct UpFilterOperator : IPngFilterOperator
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector128<byte> Invoke( public static Vector128<byte> Invoke(Vector128<byte> scan, Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft)
Vector128<byte> scan,
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft)
=> scan - above; => scan - above;
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector256<byte> Invoke( public static Vector256<byte> Invoke(Vector256<byte> scan, Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft)
Vector256<byte> scan,
Vector256<byte> left,
Vector256<byte> above,
Vector256<byte> upperLeft)
=> scan - above; => scan - above;
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector512<byte> Invoke( public static Vector512<byte> Invoke(Vector512<byte> scan, Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft)
Vector512<byte> scan,
Vector512<byte> left,
Vector512<byte> above,
Vector512<byte> upperLeft)
=> scan - above; => scan - above;
} }
@ -208,11 +172,7 @@ internal readonly struct AverageFilterOperator : IPngFilterOperator
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector128<byte> Invoke( public static Vector128<byte> Invoke(Vector128<byte> scan, Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft)
Vector128<byte> scan,
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft)
{ {
Vector128<byte> average; Vector128<byte> average;
@ -239,20 +199,18 @@ internal readonly struct AverageFilterOperator : IPngFilterOperator
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector256<byte> Invoke( public static Vector256<byte> Invoke(Vector256<byte> scan, Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft)
Vector256<byte> scan,
Vector256<byte> left, // VPAVGB rounds (left + above) / 2 upward. Complementing both inputs and
Vector256<byte> above, // the result changes that to the truncated average required by PNG.
Vector256<byte> upperLeft)
=> scan - ~Avx2.Average(~left, ~above); => scan - ~Avx2.Average(~left, ~above);
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector512<byte> Invoke( public static Vector512<byte> Invoke(Vector512<byte> scan, Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft)
Vector512<byte> scan,
Vector512<byte> left, // AVX-512BW retains VPAVGB's upward rounding, so use the same complement
Vector512<byte> above, // identity as AVX2 to obtain floor((left + above) / 2) in every byte lane.
Vector512<byte> upperLeft)
=> scan - ~Avx512BW.Average(~left, ~above); => scan - ~Avx512BW.Average(~left, ~above);
} }
@ -283,20 +241,14 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator
int distanceUpperLeft = Numerics.Abs(p - upperLeft); int distanceUpperLeft = Numerics.Abs(p - upperLeft);
// PNG resolves equal distances in left, above, upper-left order. // PNG resolves equal distances in left, above, upper-left order.
byte predictor = distanceLeft <= distanceAbove && distanceLeft <= distanceUpperLeft byte predictor = distanceLeft <= distanceAbove && distanceLeft <= distanceUpperLeft ? left : distanceAbove <= distanceUpperLeft ? above : upperLeft;
? left
: distanceAbove <= distanceUpperLeft ? above : upperLeft;
return (byte)(scan - predictor); return (byte)(scan - predictor);
} }
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector128<byte> Invoke( public static Vector128<byte> Invoke(Vector128<byte> scan, Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft)
Vector128<byte> scan,
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft)
{ {
Vector128<byte> predictor = Predict(left, above, upperLeft); Vector128<byte> predictor = Predict(left, above, upperLeft);
return scan - predictor; return scan - predictor;
@ -304,11 +256,7 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector256<byte> Invoke( public static Vector256<byte> Invoke(Vector256<byte> scan, Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft)
Vector256<byte> scan,
Vector256<byte> left,
Vector256<byte> above,
Vector256<byte> upperLeft)
{ {
Vector256<byte> predictor = Predict(left, above, upperLeft); Vector256<byte> predictor = Predict(left, above, upperLeft);
return scan - predictor; return scan - predictor;
@ -316,11 +264,7 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator
/// <inheritdoc /> /// <inheritdoc />
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static Vector512<byte> Invoke( public static Vector512<byte> Invoke(Vector512<byte> scan, Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft)
Vector512<byte> scan,
Vector512<byte> left,
Vector512<byte> above,
Vector512<byte> upperLeft)
{ {
Vector512<byte> predictor = Predict(left, above, upperLeft); Vector512<byte> predictor = Predict(left, above, upperLeft);
return scan - predictor; return scan - predictor;
@ -329,196 +273,151 @@ internal readonly struct PaethFilterOperator : IPngFilterOperator
/// <summary> /// <summary>
/// Selects the nearest Paeth neighbor for sixteen independent byte lanes. /// Selects the nearest Paeth neighbor for sixteen independent byte lanes.
/// </summary> /// </summary>
/// <param name="left">The reconstructed component immediately before each current component.</param>
/// <param name="above">The reconstructed component immediately above each current component.</param>
/// <param name="upperLeft">The reconstructed component diagonally above and before each current component.</param>
/// <returns>The selected Paeth predictor for each byte lane.</returns>
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector128<byte> Predict( private static Vector128<byte> Predict(Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft)
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft)
{ {
Vector128<byte> aboveMinusUpper = SubtractSaturate(above, upperLeft); // For p = left + above - upperLeft, the Paeth distances simplify to:
Vector128<byte> leftMinusUpper = SubtractSaturate(left, upperLeft); // distanceLeft = |above - upperLeft|
Vector128<byte> distanceLeft = SubtractSaturate(upperLeft, above) | aboveMinusUpper; // distanceAbove = |left - upperLeft|
Vector128<byte> distanceAbove = SubtractSaturate(upperLeft, left) | leftMinusUpper; // Computing both unsigned subtraction directions and OR-ing them obtains
// each absolute difference without widening the byte lanes.
return SelectPredictor( Vector128<byte> aboveMinusUpper = Vector128_.SubtractSaturate(above, upperLeft);
left, Vector128<byte> leftMinusUpper = Vector128_.SubtractSaturate(left, upperLeft);
above, Vector128<byte> distanceLeft = Vector128_.SubtractSaturate(upperLeft, above) | aboveMinusUpper;
upperLeft, Vector128<byte> distanceAbove = Vector128_.SubtractSaturate(upperLeft, left) | leftMinusUpper;
aboveMinusUpper,
leftMinusUpper, return SelectPredictor(left, above, upperLeft, aboveMinusUpper, leftMinusUpper, distanceLeft, distanceAbove);
distanceLeft,
distanceAbove);
} }
/// <summary> /// <summary>
/// Selects the nearest Paeth neighbor for thirty-two independent byte lanes. /// Selects the nearest Paeth neighbor for thirty-two independent byte lanes.
/// </summary> /// </summary>
/// <param name="left">The reconstructed component immediately before each current component.</param>
/// <param name="above">The reconstructed component immediately above each current component.</param>
/// <param name="upperLeft">The reconstructed component diagonally above and before each current component.</param>
/// <returns>The selected Paeth predictor for each byte lane.</returns>
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector256<byte> Predict( private static Vector256<byte> Predict(Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft)
Vector256<byte> left,
Vector256<byte> above,
Vector256<byte> upperLeft)
{ {
Vector256<byte> aboveMinusUpper = Avx2.SubtractSaturate(above, upperLeft); // Apply the same Paeth identities as the 128-bit path to thirty-two lanes.
Vector256<byte> leftMinusUpper = Avx2.SubtractSaturate(left, upperLeft); // Saturating subtraction in both directions forms the absolute differences
Vector256<byte> distanceLeft = Avx2.SubtractSaturate(upperLeft, above) | aboveMinusUpper; // without widening, preserving one predictor result per source byte.
Vector256<byte> distanceAbove = Avx2.SubtractSaturate(upperLeft, left) | leftMinusUpper; Vector256<byte> aboveMinusUpper = Vector256_.SubtractSaturate(above, upperLeft);
Vector256<byte> leftMinusUpper = Vector256_.SubtractSaturate(left, upperLeft);
return SelectPredictor( Vector256<byte> distanceLeft = Vector256_.SubtractSaturate(upperLeft, above) | aboveMinusUpper;
left, Vector256<byte> distanceAbove = Vector256_.SubtractSaturate(upperLeft, left) | leftMinusUpper;
above,
upperLeft, return SelectPredictor(left, above, upperLeft, aboveMinusUpper, leftMinusUpper, distanceLeft, distanceAbove);
aboveMinusUpper,
leftMinusUpper,
distanceLeft,
distanceAbove);
} }
/// <summary> /// <summary>
/// Selects the nearest Paeth neighbor for sixty-four independent byte lanes. /// Selects the nearest Paeth neighbor for sixty-four independent byte lanes.
/// </summary> /// </summary>
/// <param name="left">The reconstructed component immediately before each current component.</param>
/// <param name="above">The reconstructed component immediately above each current component.</param>
/// <param name="upperLeft">The reconstructed component diagonally above and before each current component.</param>
/// <returns>The selected Paeth predictor for each byte lane.</returns>
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector512<byte> Predict( private static Vector512<byte> Predict(Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft)
Vector512<byte> left,
Vector512<byte> above,
Vector512<byte> upperLeft)
{ {
Vector512<byte> aboveMinusUpper = Avx512BW.SubtractSaturate(above, upperLeft); // Apply the same byte-lane Paeth identities to sixty-four AVX-512BW lanes.
Vector512<byte> leftMinusUpper = Avx512BW.SubtractSaturate(left, upperLeft); // No cross-lane operation is required because every component has its own
Vector512<byte> distanceLeft = Avx512BW.SubtractSaturate(upperLeft, above) | aboveMinusUpper; // left, above, and upper-left inputs at the matching vector index.
Vector512<byte> distanceAbove = Avx512BW.SubtractSaturate(upperLeft, left) | leftMinusUpper; Vector512<byte> aboveMinusUpper = Vector512_.SubtractSaturate(above, upperLeft);
Vector512<byte> leftMinusUpper = Vector512_.SubtractSaturate(left, upperLeft);
return SelectPredictor( Vector512<byte> distanceLeft = Vector512_.SubtractSaturate(upperLeft, above) | aboveMinusUpper;
left, Vector512<byte> distanceAbove = Vector512_.SubtractSaturate(upperLeft, left) | leftMinusUpper;
above,
upperLeft, return SelectPredictor(left, above, upperLeft, aboveMinusUpper, leftMinusUpper, distanceLeft, distanceAbove);
aboveMinusUpper,
leftMinusUpper,
distanceLeft,
distanceAbove);
} }
/// <summary> /// <summary>
/// Applies Paeth distance and tie-breaking rules to sixteen lanes. /// Applies Paeth distance and tie-breaking rules to sixteen lanes.
/// </summary> /// </summary>
/// <param name="left">The left-neighbor candidates.</param>
/// <param name="above">The above-neighbor candidates.</param>
/// <param name="upperLeft">The upper-left-neighbor candidates.</param>
/// <param name="aboveMinusUpper">The saturated differences from above to upper-left.</param>
/// <param name="leftMinusUpper">The saturated differences from left to upper-left.</param>
/// <param name="distanceLeft">The Paeth distances for the left candidates.</param>
/// <param name="distanceAbove">The Paeth distances for the above candidates.</param>
/// <returns>The selected Paeth predictor for each byte lane.</returns>
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector128<byte> SelectPredictor( private static Vector128<byte> SelectPredictor(Vector128<byte> left, Vector128<byte> above, Vector128<byte> upperLeft, Vector128<byte> aboveMinusUpper, Vector128<byte> leftMinusUpper, Vector128<byte> distanceLeft, Vector128<byte> distanceAbove)
Vector128<byte> left,
Vector128<byte> above,
Vector128<byte> upperLeft,
Vector128<byte> aboveMinusUpper,
Vector128<byte> leftMinusUpper,
Vector128<byte> distanceLeft,
Vector128<byte> distanceAbove)
{ {
Vector128<byte> sameDirection = Vector128.Equals( Vector128<byte> sameDirection = Vector128.Equals(Vector128.Equals(aboveMinusUpper, Vector128<byte>.Zero), Vector128.Equals(leftMinusUpper, Vector128<byte>.Zero));
Vector128.Equals(aboveMinusUpper, Vector128<byte>.Zero),
Vector128.Equals(leftMinusUpper, Vector128<byte>.Zero));
Vector128<byte> distanceUpper = sameDirection // If left and above lie on the same side of upper-left, distanceUpper is
| SubtractSaturate(distanceAbove, distanceLeft) // their summed distance and cannot beat either neighbor; the all-bits mask
| SubtractSaturate(distanceLeft, distanceAbove); // excludes upper-left. On opposite sides, that distance is the absolute
// difference between distanceLeft and distanceAbove.
Vector128<byte> distanceUpper = sameDirection | Vector128_.SubtractSaturate(distanceAbove, distanceLeft) | Vector128_.SubtractSaturate(distanceLeft, distanceAbove);
// Equality selects above before upper-left, implementing PNG's second tie rule.
Vector128<byte> minimumAboveUpper = Vector128.Min(distanceUpper, distanceAbove); Vector128<byte> minimumAboveUpper = Vector128.Min(distanceUpper, distanceAbove);
Vector128<byte> aboveOrUpper = Vector128.ConditionalSelect( Vector128<byte> aboveOrUpper = Vector128.ConditionalSelect(Vector128.Equals(minimumAboveUpper, distanceAbove), above, upperLeft);
Vector128.Equals(minimumAboveUpper, distanceAbove),
above,
upperLeft);
// Applying the left comparison last preserves PNG's left-first tie rule. // Applying the left comparison last preserves PNG's left-first tie rule.
return Vector128.ConditionalSelect( return Vector128.ConditionalSelect(Vector128.Equals(Vector128.Min(minimumAboveUpper, distanceLeft), distanceLeft), left, aboveOrUpper);
Vector128.Equals(Vector128.Min(minimumAboveUpper, distanceLeft), distanceLeft),
left,
aboveOrUpper);
} }
/// <summary> /// <summary>
/// Applies Paeth distance and tie-breaking rules to thirty-two lanes. /// Applies Paeth distance and tie-breaking rules to thirty-two lanes.
/// </summary> /// </summary>
/// <param name="left">The left-neighbor candidates.</param>
/// <param name="above">The above-neighbor candidates.</param>
/// <param name="upperLeft">The upper-left-neighbor candidates.</param>
/// <param name="aboveMinusUpper">The saturated differences from above to upper-left.</param>
/// <param name="leftMinusUpper">The saturated differences from left to upper-left.</param>
/// <param name="distanceLeft">The Paeth distances for the left candidates.</param>
/// <param name="distanceAbove">The Paeth distances for the above candidates.</param>
/// <returns>The selected Paeth predictor for each byte lane.</returns>
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector256<byte> SelectPredictor( private static Vector256<byte> SelectPredictor(Vector256<byte> left, Vector256<byte> above, Vector256<byte> upperLeft, Vector256<byte> aboveMinusUpper, Vector256<byte> leftMinusUpper, Vector256<byte> distanceLeft, Vector256<byte> distanceAbove)
Vector256<byte> left,
Vector256<byte> above,
Vector256<byte> upperLeft,
Vector256<byte> aboveMinusUpper,
Vector256<byte> leftMinusUpper,
Vector256<byte> distanceLeft,
Vector256<byte> distanceAbove)
{ {
Vector256<byte> sameDirection = Vector256.Equals( Vector256<byte> sameDirection = Vector256.Equals(Vector256.Equals(aboveMinusUpper, Vector256<byte>.Zero), Vector256.Equals(leftMinusUpper, Vector256<byte>.Zero));
Vector256.Equals(aboveMinusUpper, Vector256<byte>.Zero),
Vector256.Equals(leftMinusUpper, Vector256<byte>.Zero));
Vector256<byte> distanceUpper = sameDirection // Exclude upper-left when its distance is the non-minimal sum; otherwise
| Avx2.SubtractSaturate(distanceAbove, distanceLeft) // compute its distance as the absolute difference of the two known distances.
| Avx2.SubtractSaturate(distanceLeft, distanceAbove); Vector256<byte> distanceUpper = sameDirection | Vector256_.SubtractSaturate(distanceAbove, distanceLeft) | Vector256_.SubtractSaturate(distanceLeft, distanceAbove);
// Select above on equality, then select left on equality to preserve PNG's
// required left, above, upper-left tie order in every byte lane.
Vector256<byte> minimumAboveUpper = Vector256.Min(distanceUpper, distanceAbove); Vector256<byte> minimumAboveUpper = Vector256.Min(distanceUpper, distanceAbove);
Vector256<byte> aboveOrUpper = Vector256.ConditionalSelect( Vector256<byte> aboveOrUpper = Vector256.ConditionalSelect(Vector256.Equals(minimumAboveUpper, distanceAbove), above, upperLeft);
Vector256.Equals(minimumAboveUpper, distanceAbove),
above, return Vector256.ConditionalSelect(Vector256.Equals(Vector256.Min(minimumAboveUpper, distanceLeft), distanceLeft), left, aboveOrUpper);
upperLeft);
return Vector256.ConditionalSelect(
Vector256.Equals(Vector256.Min(minimumAboveUpper, distanceLeft), distanceLeft),
left,
aboveOrUpper);
} }
/// <summary> /// <summary>
/// Applies Paeth distance and tie-breaking rules to sixty-four lanes. /// Applies Paeth distance and tie-breaking rules to sixty-four lanes.
/// </summary> /// </summary>
/// <param name="left">The left-neighbor candidates.</param>
/// <param name="above">The above-neighbor candidates.</param>
/// <param name="upperLeft">The upper-left-neighbor candidates.</param>
/// <param name="aboveMinusUpper">The saturated differences from above to upper-left.</param>
/// <param name="leftMinusUpper">The saturated differences from left to upper-left.</param>
/// <param name="distanceLeft">The Paeth distances for the left candidates.</param>
/// <param name="distanceAbove">The Paeth distances for the above candidates.</param>
/// <returns>The selected Paeth predictor for each byte lane.</returns>
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector512<byte> SelectPredictor( private static Vector512<byte> SelectPredictor(Vector512<byte> left, Vector512<byte> above, Vector512<byte> upperLeft, Vector512<byte> aboveMinusUpper, Vector512<byte> leftMinusUpper, Vector512<byte> distanceLeft, Vector512<byte> distanceAbove)
Vector512<byte> left,
Vector512<byte> above,
Vector512<byte> upperLeft,
Vector512<byte> aboveMinusUpper,
Vector512<byte> leftMinusUpper,
Vector512<byte> distanceLeft,
Vector512<byte> distanceAbove)
{ {
Vector512<byte> sameDirection = Vector512.Equals( Vector512<byte> sameDirection = Vector512.Equals(Vector512.Equals(aboveMinusUpper, Vector512<byte>.Zero), Vector512.Equals(leftMinusUpper, Vector512<byte>.Zero));
Vector512.Equals(aboveMinusUpper, Vector512<byte>.Zero),
Vector512.Equals(leftMinusUpper, Vector512<byte>.Zero));
Vector512<byte> distanceUpper = sameDirection // Exclude upper-left when its distance is the non-minimal sum; otherwise
| Avx512BW.SubtractSaturate(distanceAbove, distanceLeft) // compute its distance as the absolute difference of the two known distances.
| Avx512BW.SubtractSaturate(distanceLeft, distanceAbove); Vector512<byte> distanceUpper = sameDirection | Vector512_.SubtractSaturate(distanceAbove, distanceLeft) | Vector512_.SubtractSaturate(distanceLeft, distanceAbove);
// Select above on equality, then select left on equality to preserve PNG's
// required left, above, upper-left tie order in every byte lane.
Vector512<byte> minimumAboveUpper = Vector512.Min(distanceUpper, distanceAbove); Vector512<byte> minimumAboveUpper = Vector512.Min(distanceUpper, distanceAbove);
Vector512<byte> aboveOrUpper = Vector512.ConditionalSelect( Vector512<byte> aboveOrUpper = Vector512.ConditionalSelect(Vector512.Equals(minimumAboveUpper, distanceAbove), above, upperLeft);
Vector512.Equals(minimumAboveUpper, distanceAbove),
above,
upperLeft);
return Vector512.ConditionalSelect(
Vector512.Equals(Vector512.Min(minimumAboveUpper, distanceLeft), distanceLeft),
left,
aboveOrUpper);
}
/// <summary>
/// Performs an unsigned saturating subtraction using the active 128-bit instruction set.
/// </summary>
/// <param name="left">The minuend lanes.</param>
/// <param name="right">The subtrahend lanes.</param>
/// <returns>The saturated lane-wise differences.</returns>
[MethodImpl(InliningOptions.AlwaysInline)]
private static Vector128<byte> SubtractSaturate(Vector128<byte> left, Vector128<byte> right)
{
if (Sse2.IsSupported)
{
return Sse2.SubtractSaturate(left, right);
}
if (AdvSimd.IsSupported)
{
return AdvSimd.SubtractSaturate(left, right);
}
// Subtracting the smaller operand produces max(left - right, 0) without return Vector512.ConditionalSelect(Vector512.Equals(Vector512.Min(minimumAboveUpper, distanceLeft), distanceLeft), left, aboveOrUpper);
// requiring a backend-specific saturating-subtract instruction.
return left - Vector128.Min(left, right);
} }
} }

7
src/ImageSharp/Formats/Png/Filters/PaethFilter.cs

@ -192,12 +192,7 @@ internal static class PaethFilter
/// <param name="sum">The sum of the total variance of the filtered row.</param> /// <param name="sum">The sum of the total variance of the filtered row.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Encode(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, int bytesPerPixel, out int sum) public static void Encode(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, int bytesPerPixel, out int sum)
=> PngFilterEncoder.Encode<PaethFilterOperator>( => PngFilterEncoder.Encode<PaethFilterOperator>(scanline, previousScanline, result, (uint)bytesPerPixel, out sum);
scanline,
previousScanline,
result,
(uint)bytesPerPixel,
out sum);
/// <summary> /// <summary>
/// Computes a simple linear function of the three neighboring pixels (left, above, upper left), then chooses /// Computes a simple linear function of the three neighboring pixels (left, above, upper left), then chooses

121
src/ImageSharp/Formats/Png/Filters/PngFilterEncoder.cs

@ -25,12 +25,7 @@ internal static class PngFilterEncoder
// Inlining closes every static interface call over TOperator. The JIT can then remove // Inlining closes every static interface call over TOperator. The JIT can then remove
// source loads ignored by simpler predictors and specialize the active register widths. // source loads ignored by simpler predictors and specialize the active register widths.
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
public static void Encode<TOperator>( public static void Encode<TOperator>(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, uint bytesPerPixel, out int sum)
ReadOnlySpan<byte> scanline,
ReadOnlySpan<byte> previousScanline,
Span<byte> result,
uint bytesPerPixel,
out int sum)
where TOperator : struct, IPngFilterOperator where TOperator : struct, IPngFilterOperator
{ {
DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline)); DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline));
@ -51,11 +46,7 @@ internal static class PngFilterEncoder
{ {
byte above = TOperator.UsesAbove ? Unsafe.Add(ref previousBaseRef, x) : (byte)0; byte above = TOperator.UsesAbove ? Unsafe.Add(ref previousBaseRef, x) : (byte)0;
byte filtered = TOperator.Invoke( byte filtered = TOperator.Invoke(Unsafe.Add(ref scanBaseRef, x), 0, above, 0);
Unsafe.Add(ref scanBaseRef, x),
0,
above,
0);
Unsafe.Add(ref resultBaseRef, x + 1) = filtered; Unsafe.Add(ref resultBaseRef, x + 1) = filtered;
sum += Numerics.Abs(unchecked((sbyte)filtered)); sum += Numerics.Abs(unchecked((sbyte)filtered));
@ -76,28 +67,19 @@ internal static class PngFilterEncoder
// four input vectors retain the scan/left/above/upper-left PNG layout. // four input vectors retain the scan/left/above/upper-left PNG layout.
// Operator usage flags are constants after generic specialization, so // Operator usage flags are constants after generic specialization, so
// unused predictors do not retain even fault-preserving probe loads. // unused predictors do not retain even fault-preserving probe loads.
Vector512<byte> left = TOperator.UsesLeft Vector512<byte> left = TOperator.UsesLeft ? Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref scanBaseRef, xLeft)) : default;
? Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref scanBaseRef, xLeft)) Vector512<byte> above = TOperator.UsesAbove ? Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref previousBaseRef, x)) : default;
: default; Vector512<byte> upperLeft = TOperator.UsesUpperLeft ? Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref previousBaseRef, xLeft)) : default;
Vector512<byte> above = TOperator.UsesAbove
? Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref previousBaseRef, x)) Vector512<byte> filtered = TOperator.Invoke(Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref scanBaseRef, x)), left, above, upperLeft);
: default;
Vector512<byte> upperLeft = TOperator.UsesUpperLeft
? Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref previousBaseRef, xLeft))
: default;
Vector512<byte> filtered = TOperator.Invoke(
Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref scanBaseRef, x)),
left,
above,
upperLeft);
Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered; Unsafe.As<byte, Vector512<byte>>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered;
x += (uint)Vector512<byte>.Count; x += (uint)Vector512<byte>.Count;
// PNG scores each residual as abs((sbyte)residual). VPSADBW sums eight // Vector512.Abs lowers to VPABSB under the surrounding AVX-512BW guard.
// byte lanes into every other 32-bit lane without widening each byte. // Reinterpreting the signed result preserves -128's 0x80 bit pattern as
Vector512<byte> absolute = Avx512BW.Abs(filtered.AsSByte()); // the unsigned magnitude 128 consumed by VPSADBW.
Vector512<byte> absolute = Vector512.Abs(filtered.AsSByte()).AsByte();
sum512 += Avx512BW.SumAbsoluteDifferences(absolute, Vector512<byte>.Zero).AsUInt32(); sum512 += Avx512BW.SumAbsoluteDifferences(absolute, Vector512<byte>.Zero).AsUInt32();
} }
@ -114,29 +96,26 @@ internal static class PngFilterEncoder
for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector256<byte>.Count) for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector256<byte>.Count)
{ {
Vector256<byte> left = TOperator.UsesLeft // Thirty-two byte lanes preserve the same scan/left/above/upper-left
? Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref scanBaseRef, xLeft)) // correspondence as the 512-bit path. Closed operator flags remove
: default; // unused loads when the predictor does not consume that neighbor.
Vector256<byte> above = TOperator.UsesAbove Vector256<byte> left = TOperator.UsesLeft ? Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref scanBaseRef, xLeft)) : default;
? Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref previousBaseRef, x)) Vector256<byte> above = TOperator.UsesAbove ? Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref previousBaseRef, x)) : default;
: default; Vector256<byte> upperLeft = TOperator.UsesUpperLeft ? Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref previousBaseRef, xLeft)) : default;
Vector256<byte> upperLeft = TOperator.UsesUpperLeft
? Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref previousBaseRef, xLeft)) Vector256<byte> filtered = TOperator.Invoke(Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref scanBaseRef, x)), left, above, upperLeft);
: default;
Vector256<byte> filtered = TOperator.Invoke(
Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref scanBaseRef, x)),
left,
above,
upperLeft);
Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered; Unsafe.As<byte, Vector256<byte>>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered;
x += (uint)Vector256<byte>.Count; x += (uint)Vector256<byte>.Count;
Vector256<byte> absolute = Avx2.Abs(filtered.AsSByte()); // Vector256.Abs lowers to VPABSB under the surrounding AVX2 guard.
// Reinterpreting the signed result preserves -128's 0x80 bit pattern as
// the unsigned magnitude 128 consumed by VPSADBW.
Vector256<byte> absolute = Vector256.Abs(filtered.AsSByte()).AsByte();
sum256 += Avx2.SumAbsoluteDifferences(absolute, Vector256<byte>.Zero).AsUInt32(); sum256 += Avx2.SumAbsoluteDifferences(absolute, Vector256<byte>.Zero).AsUInt32();
} }
// Fold both 128-bit halves into the shared four-lane accumulator.
sum128 += sum256.GetLower() + sum256.GetUpper(); sum128 += sum256.GetLower() + sum256.GetUpper();
} }
@ -146,29 +125,24 @@ internal static class PngFilterEncoder
for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector128<byte>.Count) for (nuint xLeft = x - bytesPerPixel; (int)x <= oneRegisterFromEnd; xLeft += (uint)Vector128<byte>.Count)
{ {
Vector128<byte> left = TOperator.UsesLeft // The final vector width handles sixteen more components with the
? Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref scanBaseRef, xLeft)) // same lane-wise neighborhood layout before the scalar remainder.
: default; Vector128<byte> left = TOperator.UsesLeft ? Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref scanBaseRef, xLeft)) : default;
Vector128<byte> above = TOperator.UsesAbove Vector128<byte> above = TOperator.UsesAbove ? Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref previousBaseRef, x)) : default;
? Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref previousBaseRef, x)) Vector128<byte> upperLeft = TOperator.UsesUpperLeft ? Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref previousBaseRef, xLeft)) : default;
: default;
Vector128<byte> upperLeft = TOperator.UsesUpperLeft Vector128<byte> filtered = TOperator.Invoke(Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref scanBaseRef, x)), left, above, upperLeft);
? Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref previousBaseRef, xLeft))
: default;
Vector128<byte> filtered = TOperator.Invoke(
Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref scanBaseRef, x)),
left,
above,
upperLeft);
Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered; Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref resultBaseRef, x + 1)) = filtered;
x += (uint)Vector128<byte>.Count; x += (uint)Vector128<byte>.Count;
// AccumulateAbsolute selects the available x86 or portable widening
// reduction while preserving the same unsigned 32-bit partial sums.
sum128 = AccumulateAbsolute(sum128, filtered); sum128 = AccumulateAbsolute(sum128, filtered);
} }
} }
// Reduce the four partial lanes before adding individually scored tail bytes.
sum += unchecked((int)Vector128.Sum(sum128)); sum += unchecked((int)Vector128.Sum(sum128));
for (nuint xLeft = x - bytesPerPixel; x < (uint)scanline.Length; xLeft++, x++) for (nuint xLeft = x - bytesPerPixel; x < (uint)scanline.Length; xLeft++, x++)
@ -177,11 +151,7 @@ internal static class PngFilterEncoder
byte above = TOperator.UsesAbove ? Unsafe.Add(ref previousBaseRef, x) : (byte)0; byte above = TOperator.UsesAbove ? Unsafe.Add(ref previousBaseRef, x) : (byte)0;
byte upperLeft = TOperator.UsesUpperLeft ? Unsafe.Add(ref previousBaseRef, xLeft) : (byte)0; byte upperLeft = TOperator.UsesUpperLeft ? Unsafe.Add(ref previousBaseRef, xLeft) : (byte)0;
byte filtered = TOperator.Invoke( byte filtered = TOperator.Invoke(Unsafe.Add(ref scanBaseRef, x), left, above, upperLeft);
Unsafe.Add(ref scanBaseRef, x),
left,
above,
upperLeft);
Unsafe.Add(ref resultBaseRef, x + 1) = filtered; Unsafe.Add(ref resultBaseRef, x + 1) = filtered;
sum += Numerics.Abs(unchecked((sbyte)filtered)); sum += Numerics.Abs(unchecked((sbyte)filtered));
@ -197,27 +167,16 @@ internal static class PngFilterEncoder
[MethodImpl(InliningOptions.AlwaysInline)] [MethodImpl(InliningOptions.AlwaysInline)]
private static Vector128<uint> AccumulateAbsolute(Vector128<uint> accumulator, Vector128<byte> residuals) private static Vector128<uint> AccumulateAbsolute(Vector128<uint> accumulator, Vector128<byte> residuals)
{ {
// The generic absolute-value intrinsic selects PABSB where available and
// preserves -128's 0x80 bit pattern as the unsigned magnitude 128.
Vector128<byte> absolute = Vector128.Abs(residuals.AsSByte()).AsByte();
if (Sse2.IsSupported) if (Sse2.IsSupported)
{ {
Vector128<byte> absolute;
if (Ssse3.IsSupported)
{
absolute = Ssse3.Abs(residuals.AsSByte());
}
else
{
// SSE2 has no packed signed-byte absolute instruction. The sign mask
// implements (value + mask) XOR mask, including -128 -> 128.
Vector128<sbyte> mask = Sse2.CompareGreaterThan(Vector128<sbyte>.Zero, residuals.AsSByte());
absolute = Sse2.Xor(Sse2.Add(residuals.AsSByte(), mask), mask).AsByte();
}
return accumulator + Sse2.SumAbsoluteDifferences(absolute, Vector128<byte>.Zero).AsUInt32(); return accumulator + Sse2.SumAbsoluteDifferences(absolute, Vector128<byte>.Zero).AsUInt32();
} }
Vector128<byte> absoluteArm = Vector128.Abs(residuals.AsSByte()).AsByte(); (Vector128<ushort> lower16, Vector128<ushort> upper16) = Vector128.Widen(absolute);
(Vector128<ushort> lower16, Vector128<ushort> upper16) = Vector128.Widen(absoluteArm);
(Vector128<uint> lower0, Vector128<uint> lower1) = Vector128.Widen(lower16); (Vector128<uint> lower0, Vector128<uint> lower1) = Vector128.Widen(lower16);
(Vector128<uint> upper0, Vector128<uint> upper1) = Vector128.Widen(upper16); (Vector128<uint> upper0, Vector128<uint> upper1) = Vector128.Widen(upper16);

12
src/ImageSharp/Formats/Png/Filters/SubFilter.cs

@ -102,7 +102,7 @@ internal static class SubFilter
} }
/// <summary> /// <summary>
/// Encodes a scanline with the sup filter applied. /// Encodes a scanline with the sub filter applied.
/// </summary> /// </summary>
/// <param name="scanline">The scanline to encode.</param> /// <param name="scanline">The scanline to encode.</param>
/// <param name="result">The filtered scanline result.</param> /// <param name="result">The filtered scanline result.</param>
@ -110,10 +110,8 @@ internal static class SubFilter
/// <param name="sum">The sum of the total variance of the filtered row.</param> /// <param name="sum">The sum of the total variance of the filtered row.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Encode(ReadOnlySpan<byte> scanline, Span<byte> result, int bytesPerPixel, out int sum) public static void Encode(ReadOnlySpan<byte> scanline, Span<byte> result, int bytesPerPixel, out int sum)
=> PngFilterEncoder.Encode<SubFilterOperator>(
scanline, // Sub does not consume an above neighbor, so the shared traversal may alias
scanline, // the unused previous-row argument to the current row without an extra buffer.
result, => PngFilterEncoder.Encode<SubFilterOperator>(scanline, scanline, result, (uint)bytesPerPixel, out sum);
(uint)bytesPerPixel,
out sum);
} }

10
src/ImageSharp/Formats/Png/Filters/UpFilter.cs

@ -36,10 +36,8 @@ internal static class UpFilter
/// <param name="sum">The sum of the total variance of the filtered row.</param> /// <param name="sum">The sum of the total variance of the filtered row.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Encode(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, out int sum) public static void Encode(ReadOnlySpan<byte> scanline, ReadOnlySpan<byte> previousScanline, Span<byte> result, out int sum)
=> PngFilterEncoder.Encode<UpFilterOperator>(
scanline, // Up never reads a left neighbor, so bytesPerPixel is deliberately zero in
previousScanline, // the shared traversal and no unsigned left offset is evaluated.
result, => PngFilterEncoder.Encode<UpFilterOperator>(scanline, previousScanline, result, 0, out sum);
0,
out sum);
} }

1
src/ImageSharp/Formats/Webp/AlphaDecoder.cs

@ -363,6 +363,7 @@ internal class AlphaDecoder : IDisposable
} }
else else
{ {
// Byte addition intentionally wraps modulo 256, matching the WebP alpha predictor.
TensorPrimitives_.Add(input[..width], prev[..width], dst[..width]); TensorPrimitives_.Add(input[..width], prev[..width], dst[..width]);
} }
} }

18
src/ImageSharp/Formats/Webp/Lossless/Vp8LHistogram.cs

@ -331,7 +331,7 @@ internal abstract unsafe class Vp8LHistogram
{ {
if (b.IsUsed(0)) if (b.IsUsed(0))
{ {
AddVector(this.Literal, b.Literal, output.Literal, literalSize); TensorPrimitives_.Add(this.Literal[..literalSize], b.Literal[..literalSize], output.Literal[..literalSize]);
} }
else else
{ {
@ -354,7 +354,7 @@ internal abstract unsafe class Vp8LHistogram
{ {
if (b.IsUsed(1)) if (b.IsUsed(1))
{ {
AddVector(this.Red, b.Red, output.Red, size); TensorPrimitives_.Add(this.Red[..size], b.Red[..size], output.Red[..size]);
} }
else else
{ {
@ -377,7 +377,7 @@ internal abstract unsafe class Vp8LHistogram
{ {
if (b.IsUsed(2)) if (b.IsUsed(2))
{ {
AddVector(this.Blue, b.Blue, output.Blue, size); TensorPrimitives_.Add(this.Blue[..size], b.Blue[..size], output.Blue[..size]);
} }
else else
{ {
@ -400,7 +400,7 @@ internal abstract unsafe class Vp8LHistogram
{ {
if (b.IsUsed(3)) if (b.IsUsed(3))
{ {
AddVector(this.Alpha, b.Alpha, output.Alpha, size); TensorPrimitives_.Add(this.Alpha[..size], b.Alpha[..size], output.Alpha[..size]);
} }
else else
{ {
@ -423,7 +423,7 @@ internal abstract unsafe class Vp8LHistogram
{ {
if (b.IsUsed(4)) if (b.IsUsed(4))
{ {
AddVector(this.Distance, b.Distance, output.Distance, size); TensorPrimitives_.Add(this.Distance[..size], b.Distance[..size], output.Distance[..size]);
} }
else else
{ {
@ -534,14 +534,6 @@ internal abstract unsafe class Vp8LHistogram
return cost; return cost;
} }
private static void AddVector(Span<uint> a, Span<uint> b, Span<uint> output, int count)
{
DebugGuard.MustBeGreaterThanOrEqualTo(a.Length, count, nameof(a.Length));
DebugGuard.MustBeGreaterThanOrEqualTo(b.Length, count, nameof(b.Length));
DebugGuard.MustBeGreaterThanOrEqualTo(output.Length, count, nameof(output.Length));
TensorPrimitives_.Add(a[..count], b[..count], output[..count]);
}
} }
internal sealed unsafe class OwnedVp8LHistogram : Vp8LHistogram, IDisposable internal sealed unsafe class OwnedVp8LHistogram : Vp8LHistogram, IDisposable

1080
src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPixelBlenders.Generated.cs

File diff suppressed because it is too large

10
src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPixelBlenders.Generated.tt

@ -66,17 +66,11 @@ foreach (var composer in composers)
=> AssociatedAlphaPorterDuffFunctions.<#= blenderComposer #>(background, source, amount); => AssociatedAlphaPorterDuffFunctions.<#= blenderComposer #>(background, source, amount);
/// <inheritdoc /> /// <inheritdoc />
public static Vector256<float> Invoke( public static Vector256<float> Invoke(Vector256<float> background, Vector256<float> source, Vector256<float> amount)
Vector256<float> background,
Vector256<float> source,
Vector256<float> amount)
=> AssociatedAlphaPorterDuffFunctions.<#= blenderComposer #>(background, source, amount); => AssociatedAlphaPorterDuffFunctions.<#= blenderComposer #>(background, source, amount);
/// <inheritdoc /> /// <inheritdoc />
public static Vector512<float> Invoke( public static Vector512<float> Invoke(Vector512<float> background, Vector512<float> source, Vector512<float> amount)
Vector512<float> background,
Vector512<float> source,
Vector512<float> amount)
=> AssociatedAlphaPorterDuffFunctions.<#= blenderComposer #>(background, source, amount); => AssociatedAlphaPorterDuffFunctions.<#= blenderComposer #>(background, source, amount);
} }

5
src/ImageSharp/PixelFormats/PixelBlenders/AssociatedAlphaPixelBlender{TPixel,TOperator}.cs

@ -20,10 +20,7 @@ internal abstract class AssociatedAlphaPixelBlender<TPixel, TOperator> : PixelBl
public sealed override TPixel Blend(TPixel background, TPixel source, float amount) public sealed override TPixel Blend(TPixel background, TPixel source, float amount)
{ {
// Associated RGB and alpha must remain in their stored representation throughout composition. // Associated RGB and alpha must remain in their stored representation throughout composition.
Vector4 result = TOperator.Invoke( Vector4 result = TOperator.Invoke(background.ToAssociatedScaledVector4(), source.ToAssociatedScaledVector4(), Numerics.Clamp(amount, 0, 1F));
background.ToAssociatedScaledVector4(),
source.ToAssociatedScaledVector4(),
Numerics.Clamp(amount, 0, 1F));
return TPixel.FromAssociatedScaledVector4(result); return TPixel.FromAssociatedScaledVector4(result);
} }

1080
src/ImageSharp/PixelFormats/PixelBlenders/DefaultPixelBlenders.Generated.cs

File diff suppressed because it is too large

10
src/ImageSharp/PixelFormats/PixelBlenders/DefaultPixelBlenders.Generated.tt

@ -66,17 +66,11 @@ foreach (var composer in composers)
=> PorterDuffFunctions.<#= blenderComposer #>(background, source, amount); => PorterDuffFunctions.<#= blenderComposer #>(background, source, amount);
/// <inheritdoc /> /// <inheritdoc />
public static Vector256<float> Invoke( public static Vector256<float> Invoke(Vector256<float> background, Vector256<float> source, Vector256<float> amount)
Vector256<float> background,
Vector256<float> source,
Vector256<float> amount)
=> PorterDuffFunctions.<#= blenderComposer #>(background, source, amount); => PorterDuffFunctions.<#= blenderComposer #>(background, source, amount);
/// <inheritdoc /> /// <inheritdoc />
public static Vector512<float> Invoke( public static Vector512<float> Invoke(Vector512<float> background, Vector512<float> source, Vector512<float> amount)
Vector512<float> background,
Vector512<float> source,
Vector512<float> amount)
=> PorterDuffFunctions.<#= blenderComposer #>(background, source, amount); => PorterDuffFunctions.<#= blenderComposer #>(background, source, amount);
} }

10
src/ImageSharp/PixelFormats/PixelBlenders/IPixelBlenderOperator.cs

@ -27,10 +27,7 @@ internal interface IPixelBlenderOperator
/// <param name="source">The source RGBA lanes.</param> /// <param name="source">The source RGBA lanes.</param>
/// <param name="amount">The source opacity repeated across each pixel's four lanes.</param> /// <param name="amount">The source opacity repeated across each pixel's four lanes.</param>
/// <returns>The blended RGBA lanes.</returns> /// <returns>The blended RGBA lanes.</returns>
public static abstract Vector256<float> Invoke( public static abstract Vector256<float> Invoke(Vector256<float> background, Vector256<float> source, Vector256<float> amount);
Vector256<float> background,
Vector256<float> source,
Vector256<float> amount);
/// <summary> /// <summary>
/// Blends four pixels represented by four consecutive groups of four RGBA lanes. /// Blends four pixels represented by four consecutive groups of four RGBA lanes.
@ -39,8 +36,5 @@ internal interface IPixelBlenderOperator
/// <param name="source">The source RGBA lanes.</param> /// <param name="source">The source RGBA lanes.</param>
/// <param name="amount">The source opacity repeated across each pixel's four lanes.</param> /// <param name="amount">The source opacity repeated across each pixel's four lanes.</param>
/// <returns>The blended RGBA lanes.</returns> /// <returns>The blended RGBA lanes.</returns>
public static abstract Vector512<float> Invoke( public static abstract Vector512<float> Invoke(Vector512<float> background, Vector512<float> source, Vector512<float> amount);
Vector512<float> background,
Vector512<float> source,
Vector512<float> amount);
} }

302
src/ImageSharp/PixelFormats/PixelBlenders/PixelBlender{TPixel,TOperator}.cs

@ -5,7 +5,6 @@ using System.Numerics;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Runtime.Intrinsics; using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace SixLabors.ImageSharp.PixelFormats.PixelBlenders; namespace SixLabors.ImageSharp.PixelFormats.PixelBlenders;
@ -19,11 +18,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
where TOperator : struct, IPixelBlenderOperator where TOperator : struct, IPixelBlenderOperator
{ {
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendFunction( protected sealed override void BlendFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, ReadOnlySpan<Vector4> source, float amount)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
float amount)
{ {
// Public entry points validate the row lengths, so all three references can advance in lockstep. // Public entry points validate the row lengths, so all three references can advance in lockstep.
int scalarStart = 0; int scalarStart = 0;
@ -33,7 +28,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background); ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background);
ref Vector4 sourceRef = ref MemoryMarshal.GetReference(source); ref Vector4 sourceRef = ref MemoryMarshal.GetReference(source);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
// A 512-bit register holds four complete RGBA pixels in [R,G,B,A] groups. // A 512-bit register holds four complete RGBA pixels in [R,G,B,A] groups.
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
@ -44,15 +39,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (nuint i = 0; i < (uint)vectorCount; i++) for (nuint i = 0; i < (uint)vectorCount; i++)
{ {
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), Unsafe.Add(ref sourceBase, i), amountVector);
Unsafe.Add(ref backgroundBase, i),
Unsafe.Add(ref sourceBase, i),
amountVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
// A 256-bit register holds two complete RGBA pixels in [R,G,B,A] groups. // A 256-bit register holds two complete RGBA pixels in [R,G,B,A] groups.
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
@ -63,10 +55,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (nuint i = 0; i < (uint)vectorCount; i++) for (nuint i = 0; i < (uint)vectorCount; i++)
{ {
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), Unsafe.Add(ref sourceBase, i), amountVector);
Unsafe.Add(ref backgroundBase, i),
Unsafe.Add(ref sourceBase, i),
amountVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -75,19 +64,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
// Vector4 is both the scalar pixel representation and the portable SIMD fallback. // Vector4 is both the scalar pixel representation and the portable SIMD fallback.
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke( Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke(Unsafe.Add(ref backgroundRef, (uint)i), Unsafe.Add(ref sourceRef, (uint)i), amount);
Unsafe.Add(ref backgroundRef, (uint)i),
Unsafe.Add(ref sourceRef, (uint)i),
amount);
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendFunction( protected sealed override void BlendFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, Vector4 source, float amount)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
float amount)
{ {
// Public entry points validate the row lengths, so the destination and background advance together. // Public entry points validate the row lengths, so the destination and background advance together.
int scalarStart = 0; int scalarStart = 0;
@ -96,7 +78,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector4 destinationRef = ref MemoryMarshal.GetReference(destination); ref Vector4 destinationRef = ref MemoryMarshal.GetReference(destination);
ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background); ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
// Repeat one [R,G,B,A] source group four times to match the four background pixels. // Repeat one [R,G,B,A] source group four times to match the four background pixels.
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
@ -107,15 +89,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (nuint i = 0; i < (uint)vectorCount; i++) for (nuint i = 0; i < (uint)vectorCount; i++)
{ {
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), sourceVector, amountVector);
Unsafe.Add(ref backgroundBase, i),
sourceVector,
amountVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
// Repeat one [R,G,B,A] source group twice to match the two background pixels. // Repeat one [R,G,B,A] source group twice to match the two background pixels.
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
@ -126,10 +105,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (nuint i = 0; i < (uint)vectorCount; i++) for (nuint i = 0; i < (uint)vectorCount; i++)
{ {
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), sourceVector, amountVector);
Unsafe.Add(ref backgroundBase, i),
sourceVector,
amountVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -138,19 +114,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
// The remaining pixel count is at most three after AVX-512 or one after AVX2. // The remaining pixel count is at most three after AVX-512 or one after AVX2.
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke( Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke(Unsafe.Add(ref backgroundRef, (uint)i), source, amount);
Unsafe.Add(ref backgroundRef, (uint)i),
source,
amount);
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendFunction( protected sealed override void BlendFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, ReadOnlySpan<Vector4> source, ReadOnlySpan<float> amount)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
ReadOnlySpan<float> amount)
{ {
// Each amount belongs to one pixel and must be repeated across that pixel's four RGBA lanes. // Each amount belongs to one pixel and must be repeated across that pixel's four RGBA lanes.
int scalarStart = 0; int scalarStart = 0;
@ -160,7 +129,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector4 sourceRef = ref MemoryMarshal.GetReference(source); ref Vector4 sourceRef = ref MemoryMarshal.GetReference(source);
ref float amountRef = ref MemoryMarshal.GetReference(amount); ref float amountRef = ref MemoryMarshal.GetReference(amount);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef); ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef);
@ -173,15 +142,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float amountBase = ref Unsafe.Add(ref amountRef, i * 4); ref float amountBase = ref Unsafe.Add(ref amountRef, i * 4);
Vector512<float> amountVector = CreateClampedVector512(ref amountBase); Vector512<float> amountVector = CreateClampedVector512(ref amountBase);
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), Unsafe.Add(ref sourceBase, i), amountVector);
Unsafe.Add(ref backgroundBase, i),
Unsafe.Add(ref sourceBase, i),
amountVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef); ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef);
@ -194,10 +160,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float amountBase = ref Unsafe.Add(ref amountRef, i * 2); ref float amountBase = ref Unsafe.Add(ref amountRef, i * 2);
Vector256<float> amountVector = CreateClampedVector256(ref amountBase); Vector256<float> amountVector = CreateClampedVector256(ref amountBase);
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), Unsafe.Add(ref sourceBase, i), amountVector);
Unsafe.Add(ref backgroundBase, i),
Unsafe.Add(ref sourceBase, i),
amountVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -205,19 +168,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke( Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke(Unsafe.Add(ref backgroundRef, (uint)i), Unsafe.Add(ref sourceRef, (uint)i), Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
Unsafe.Add(ref backgroundRef, (uint)i),
Unsafe.Add(ref sourceRef, (uint)i),
Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendFunction( protected sealed override void BlendFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, Vector4 source, ReadOnlySpan<float> amount)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
ReadOnlySpan<float> amount)
{ {
// The source is invariant, while each background pixel has its own independently clamped amount. // The source is invariant, while each background pixel has its own independently clamped amount.
int scalarStart = 0; int scalarStart = 0;
@ -226,7 +182,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background); ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background);
ref float amountRef = ref MemoryMarshal.GetReference(amount); ref float amountRef = ref MemoryMarshal.GetReference(amount);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef); ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef);
@ -238,15 +194,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float amountBase = ref Unsafe.Add(ref amountRef, i * 4); ref float amountBase = ref Unsafe.Add(ref amountRef, i * 4);
Vector512<float> amountVector = CreateClampedVector512(ref amountBase); Vector512<float> amountVector = CreateClampedVector512(ref amountBase);
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), sourceVector, amountVector);
Unsafe.Add(ref backgroundBase, i),
sourceVector,
amountVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef); ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef);
@ -258,10 +211,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float amountBase = ref Unsafe.Add(ref amountRef, i * 2); ref float amountBase = ref Unsafe.Add(ref amountRef, i * 2);
Vector256<float> amountVector = CreateClampedVector256(ref amountBase); Vector256<float> amountVector = CreateClampedVector256(ref amountBase);
Unsafe.Add(ref destinationBase, i) = TOperator.Invoke( Unsafe.Add(ref destinationBase, i) = TOperator.Invoke(Unsafe.Add(ref backgroundBase, i), sourceVector, amountVector);
Unsafe.Add(ref backgroundBase, i),
sourceVector,
amountVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -269,20 +219,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke( Unsafe.Add(ref destinationRef, (uint)i) = TOperator.Invoke(Unsafe.Add(ref backgroundRef, (uint)i), source, Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
Unsafe.Add(ref backgroundRef, (uint)i),
source,
Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendWithCoverageFunction( protected sealed override void BlendWithCoverageFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, ReadOnlySpan<Vector4> source, float amount, ReadOnlySpan<float> coverage)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
float amount,
ReadOnlySpan<float> coverage)
{ {
// Coverage mixes the composed result back toward the original background, so it is fused into this pass. // Coverage mixes the composed result back toward the original background, so it is fused into this pass.
int scalarStart = 0; int scalarStart = 0;
@ -293,7 +235,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector4 sourceRef = ref MemoryMarshal.GetReference(source); ref Vector4 sourceRef = ref MemoryMarshal.GetReference(source);
ref float coverageRef = ref MemoryMarshal.GetReference(coverage); ref float coverageRef = ref MemoryMarshal.GetReference(coverage);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef); ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef);
@ -306,20 +248,14 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector512<float> backgroundVector = ref Unsafe.Add(ref backgroundBase, i); ref Vector512<float> backgroundVector = ref Unsafe.Add(ref backgroundBase, i);
ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 4); ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 4);
Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase); Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase);
Vector512<float> blended = TOperator.Invoke( Vector512<float> blended = TOperator.Invoke(backgroundVector, Unsafe.Add(ref sourceBase, i), amountVector);
backgroundVector,
Unsafe.Add(ref sourceBase, i), Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef); ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef);
@ -332,15 +268,9 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector256<float> backgroundVector = ref Unsafe.Add(ref backgroundBase, i); ref Vector256<float> backgroundVector = ref Unsafe.Add(ref backgroundBase, i);
ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 2); ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 2);
Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase); Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase);
Vector256<float> blended = TOperator.Invoke( Vector256<float> blended = TOperator.Invoke(backgroundVector, Unsafe.Add(ref sourceBase, i), amountVector);
backgroundVector,
Unsafe.Add(ref sourceBase, i), Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -349,25 +279,14 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i); Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i);
Vector4 blended = TOperator.Invoke( Vector4 blended = TOperator.Invoke(backgroundPixel, Unsafe.Add(ref sourceRef, (uint)i), amount);
backgroundPixel,
Unsafe.Add(ref sourceRef, (uint)i), Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(backgroundPixel, blended, Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
amount);
Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(
backgroundPixel,
blended,
Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendWithCoverageFunction( protected sealed override void BlendWithCoverageFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, Vector4 source, float amount, ReadOnlySpan<float> coverage)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
float amount,
ReadOnlySpan<float> coverage)
{ {
// The constant source is expanded once per selected width and reused for the complete row. // The constant source is expanded once per selected width and reused for the complete row.
int scalarStart = 0; int scalarStart = 0;
@ -377,7 +296,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background); ref Vector4 backgroundRef = ref MemoryMarshal.GetReference(background);
ref float coverageRef = ref MemoryMarshal.GetReference(coverage); ref float coverageRef = ref MemoryMarshal.GetReference(coverage);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef); ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef);
@ -392,15 +311,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase); Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase);
Vector512<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector); Vector512<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage( Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef); ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef);
@ -415,10 +331,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase); Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase);
Vector256<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector); Vector256<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage( Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -429,20 +342,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i); Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i);
Vector4 blended = TOperator.Invoke(backgroundPixel, source, amount); Vector4 blended = TOperator.Invoke(backgroundPixel, source, amount);
Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage( Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(backgroundPixel, blended, Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
backgroundPixel,
blended,
Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendWithCoverageFunction( protected sealed override void BlendWithCoverageFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, ReadOnlySpan<Vector4> source, ReadOnlySpan<float> amount, ReadOnlySpan<float> coverage)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
ReadOnlySpan<float> amount,
ReadOnlySpan<float> coverage)
{ {
// Amount controls composition while coverage controls the final mix with the untouched background. // Amount controls composition while coverage controls the final mix with the untouched background.
int scalarStart = 0; int scalarStart = 0;
@ -453,7 +358,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float amountRef = ref MemoryMarshal.GetReference(amount); ref float amountRef = ref MemoryMarshal.GetReference(amount);
ref float coverageRef = ref MemoryMarshal.GetReference(coverage); ref float coverageRef = ref MemoryMarshal.GetReference(coverage);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef); ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef);
@ -467,20 +372,14 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 4); ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 4);
Vector512<float> amountVector = CreateClampedVector512(ref amountBase); Vector512<float> amountVector = CreateClampedVector512(ref amountBase);
Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase); Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase);
Vector512<float> blended = TOperator.Invoke( Vector512<float> blended = TOperator.Invoke(backgroundVector, Unsafe.Add(ref sourceBase, i), amountVector);
backgroundVector,
Unsafe.Add(ref sourceBase, i), Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef); ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef);
@ -494,15 +393,9 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 2); ref float coverageBase = ref Unsafe.Add(ref coverageRef, i * 2);
Vector256<float> amountVector = CreateClampedVector256(ref amountBase); Vector256<float> amountVector = CreateClampedVector256(ref amountBase);
Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase); Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase);
Vector256<float> blended = TOperator.Invoke( Vector256<float> blended = TOperator.Invoke(backgroundVector, Unsafe.Add(ref sourceBase, i), amountVector);
backgroundVector,
Unsafe.Add(ref sourceBase, i), Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -511,25 +404,14 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i); Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i);
Vector4 blended = TOperator.Invoke( Vector4 blended = TOperator.Invoke(backgroundPixel, Unsafe.Add(ref sourceRef, (uint)i), Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
backgroundPixel,
Unsafe.Add(ref sourceRef, (uint)i), Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(backgroundPixel, blended, Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(
backgroundPixel,
blended,
Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
} }
} }
/// <inheritdoc /> /// <inheritdoc />
protected sealed override void BlendWithCoverageFunction( protected sealed override void BlendWithCoverageFunction(Span<Vector4> destination, ReadOnlySpan<Vector4> background, Vector4 source, ReadOnlySpan<float> amount, ReadOnlySpan<float> coverage)
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
ReadOnlySpan<float> amount,
ReadOnlySpan<float> coverage)
{ {
// The invariant source is expanded once; only amount and coverage are gathered for each vector batch. // The invariant source is expanded once; only amount and coverage are gathered for each vector batch.
int scalarStart = 0; int scalarStart = 0;
@ -539,7 +421,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
ref float amountRef = ref MemoryMarshal.GetReference(amount); ref float amountRef = ref MemoryMarshal.GetReference(amount);
ref float coverageRef = ref MemoryMarshal.GetReference(coverage); ref float coverageRef = ref MemoryMarshal.GetReference(coverage);
if (Avx512F.IsSupported && destination.Length >= 4) if (Vector512.IsHardwareAccelerated && destination.Length >= 4)
{ {
ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef); ref Vector512<float> destinationBase = ref Unsafe.As<Vector4, Vector512<float>>(ref destinationRef);
ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef); ref Vector512<float> backgroundBase = ref Unsafe.As<Vector4, Vector512<float>>(ref backgroundRef);
@ -555,15 +437,12 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase); Vector512<float> coverageVector = CreateClampedVector512(ref coverageBase);
Vector512<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector); Vector512<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage( Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 4; scalarStart = vectorCount * 4;
} }
else if (Avx2.IsSupported && destination.Length >= 2) else if (Vector256.IsHardwareAccelerated && destination.Length >= 2)
{ {
ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef); ref Vector256<float> destinationBase = ref Unsafe.As<Vector4, Vector256<float>>(ref destinationRef);
ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef); ref Vector256<float> backgroundBase = ref Unsafe.As<Vector4, Vector256<float>>(ref backgroundRef);
@ -579,10 +458,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase); Vector256<float> coverageVector = CreateClampedVector256(ref coverageBase);
Vector256<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector); Vector256<float> blended = TOperator.Invoke(backgroundVector, sourceVector, amountVector);
Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage( Unsafe.Add(ref destinationBase, i) = PorterDuffFunctions.BlendWithCoverage(backgroundVector, blended, coverageVector);
backgroundVector,
blended,
coverageVector);
} }
scalarStart = vectorCount * 2; scalarStart = vectorCount * 2;
@ -591,15 +467,9 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
for (int i = scalarStart; i < destination.Length; i++) for (int i = scalarStart; i < destination.Length; i++)
{ {
Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i); Vector4 backgroundPixel = Unsafe.Add(ref backgroundRef, (uint)i);
Vector4 blended = TOperator.Invoke( Vector4 blended = TOperator.Invoke(backgroundPixel, source, Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
backgroundPixel,
source, Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(backgroundPixel, blended, Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
Numerics.Clamp(Unsafe.Add(ref amountRef, (uint)i), 0, 1F));
Unsafe.Add(ref destinationRef, (uint)i) = PorterDuffFunctions.BlendWithCoverage(
backgroundPixel,
blended,
Numerics.Clamp(Unsafe.Add(ref coverageRef, (uint)i), 0, 1F));
} }
} }
@ -619,23 +489,7 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
/// <returns>Four consecutive copies of the pixel.</returns> /// <returns>Four consecutive copies of the pixel.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<float> CreateVector512(Vector4 pixel) private static Vector512<float> CreateVector512(Vector4 pixel)
=> Vector512.Create( => Vector512.Create(pixel.X, pixel.Y, pixel.Z, pixel.W, pixel.X, pixel.Y, pixel.Z, pixel.W, pixel.X, pixel.Y, pixel.Z, pixel.W, pixel.X, pixel.Y, pixel.Z, pixel.W);
pixel.X,
pixel.Y,
pixel.Z,
pixel.W,
pixel.X,
pixel.Y,
pixel.Z,
pixel.W,
pixel.X,
pixel.Y,
pixel.Z,
pixel.W,
pixel.X,
pixel.Y,
pixel.Z,
pixel.W);
/// <summary> /// <summary>
/// Expands and clamps two per-pixel scalar values for two packed RGBA pixels. /// Expands and clamps two per-pixel scalar values for two packed RGBA pixels.
@ -645,12 +499,10 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<float> CreateClampedVector256(ref float values) private static Vector256<float> CreateClampedVector256(ref float values)
{ {
Vector256<float> result = Vector256.Create( Vector256<float> result = Vector256.Create(Vector128.Create(values), Vector128.Create(Unsafe.Add(ref values, 1)));
Vector128.Create(values),
Vector128.Create(Unsafe.Add(ref values, 1)));
// Amount and coverage share the same public 0..1 contract and therefore the same packed clamp. // Amount and coverage share the same public 0..1 contract and therefore the same packed clamp.
return Avx.Min(Avx.Max(Vector256<float>.Zero, result), Vector256.Create(1F)); return Vector256.Min(Vector256.Max(Vector256<float>.Zero, result), Vector256.Create(1F));
} }
/// <summary> /// <summary>
@ -661,24 +513,9 @@ internal abstract class PixelBlender<TPixel, TOperator> : PixelBlender<TPixel>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<float> CreateClampedVector512(ref float values) private static Vector512<float> CreateClampedVector512(ref float values)
{ {
Vector512<float> result = Vector512.Create( Vector512<float> result = Vector512.Create(values, values, values, values, Unsafe.Add(ref values, 1), Unsafe.Add(ref values, 1), Unsafe.Add(ref values, 1), Unsafe.Add(ref values, 1), Unsafe.Add(ref values, 2), Unsafe.Add(ref values, 2), Unsafe.Add(ref values, 2), Unsafe.Add(ref values, 2), Unsafe.Add(ref values, 3), Unsafe.Add(ref values, 3), Unsafe.Add(ref values, 3), Unsafe.Add(ref values, 3));
values,
values,
values,
values,
Unsafe.Add(ref values, 1),
Unsafe.Add(ref values, 1),
Unsafe.Add(ref values, 1),
Unsafe.Add(ref values, 1),
Unsafe.Add(ref values, 2),
Unsafe.Add(ref values, 2),
Unsafe.Add(ref values, 2),
Unsafe.Add(ref values, 2),
Unsafe.Add(ref values, 3),
Unsafe.Add(ref values, 3),
Unsafe.Add(ref values, 3),
Unsafe.Add(ref values, 3));
// Amount and coverage share the same public 0..1 contract and therefore the same packed clamp.
return Vector512.Min(Vector512.Max(Vector512<float>.Zero, result), Vector512.Create(1F)); return Vector512.Min(Vector512.Max(Vector512<float>.Zero, result), Vector512.Create(1F));
} }
} }
@ -696,10 +533,7 @@ internal abstract class DefaultPixelBlender<TPixel, TOperator> : PixelBlender<TP
public sealed override TPixel Blend(TPixel background, TPixel source, float amount) public sealed override TPixel Blend(TPixel background, TPixel source, float amount)
{ {
// The operator consumes the same unassociated, scaled representation used by the bulk conversion path. // The operator consumes the same unassociated, scaled representation used by the bulk conversion path.
Vector4 result = TOperator.Invoke( Vector4 result = TOperator.Invoke(background.ToUnassociatedScaledVector4(), source.ToUnassociatedScaledVector4(), Numerics.Clamp(amount, 0, 1F));
background.ToUnassociatedScaledVector4(),
source.ToUnassociatedScaledVector4(),
Numerics.Clamp(amount, 0, 1F));
return TPixel.FromUnassociatedScaledVector4(result); return TPixel.FromUnassociatedScaledVector4(result);
} }

9
src/ImageSharp/PixelFormats/Utils/Vector4Converters.Affine.cs

@ -52,8 +52,7 @@ internal static partial class Vector4Converters
for (; index <= oneRegisterFromEnd; index += vectorsPerRegister) for (; index <= oneRegisterFromEnd; index += vectorsPerRegister)
{ {
ref Vector512<float> vector = ref Unsafe.As<Vector4, Vector512<float>>( ref Vector512<float> vector = ref Unsafe.As<Vector4, Vector512<float>>(ref Unsafe.Add(ref vectorBase, (uint)index));
ref Unsafe.Add(ref vectorBase, (uint)index));
vector = transform.Invoke(vector); vector = transform.Invoke(vector);
} }
@ -66,8 +65,7 @@ internal static partial class Vector4Converters
for (; index <= oneRegisterFromEnd; index += vectorsPerRegister) for (; index <= oneRegisterFromEnd; index += vectorsPerRegister)
{ {
ref Vector256<float> vector = ref Unsafe.As<Vector4, Vector256<float>>( ref Vector256<float> vector = ref Unsafe.As<Vector4, Vector256<float>>(ref Unsafe.Add(ref vectorBase, (uint)index));
ref Unsafe.Add(ref vectorBase, (uint)index));
vector = transform.Invoke(vector); vector = transform.Invoke(vector);
} }
@ -79,8 +77,7 @@ internal static partial class Vector4Converters
// consumes every remaining complete pixel and leaves no scalar remainder. // consumes every remaining complete pixel and leaves no scalar remainder.
for (; index < vectors.Length; index++) for (; index < vectors.Length; index++)
{ {
ref Vector128<float> vector = ref Unsafe.As<Vector4, Vector128<float>>( ref Vector128<float> vector = ref Unsafe.As<Vector4, Vector128<float>>(ref Unsafe.Add(ref vectorBase, (uint)index));
ref Unsafe.Add(ref vectorBase, (uint)index));
vector = transform.Invoke(vector); vector = transform.Invoke(vector);
} }

8
src/ImageSharp/PixelFormats/Utils/Vector4Converters.AffineOperators.cs

@ -73,9 +73,7 @@ internal static partial class Vector4Converters
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public Vector4 Invoke(Vector4 source) public Vector4 Invoke(Vector4 source)
{ {
Vector128<float> result = Vector128<float> result = (source.AsVector128() * this.multiplier.GetLower().GetLower()) + this.offset.GetLower().GetLower();
(source.AsVector128() * this.multiplier.GetLower().GetLower())
+ this.offset.GetLower().GetLower();
return result.AsVector4(); return result.AsVector4();
} }
@ -126,9 +124,7 @@ internal static partial class Vector4Converters
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public Vector4 Invoke(Vector4 source) public Vector4 Invoke(Vector4 source)
{ {
Vector128<float> result = Vector128<float> result = (source.AsVector128() + this.offset.GetLower().GetLower()) / this.divisor.GetLower().GetLower();
(source.AsVector128() + this.offset.GetLower().GetLower())
/ this.divisor.GetLower().GetLower();
return result.AsVector4(); return result.AsVector4();
} }

3
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/CmykColorConversion.cs

@ -9,8 +9,7 @@ namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
[Config(typeof(Config.Short))] [Config(typeof(Config.Short))]
public class CmykColorConversion : ColorConversionBenchmark public class CmykColorConversion : ColorConversionBenchmark
{ {
private readonly JpegColorConverterBase converter = private readonly JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(JpegColorSpace.Cmyk, 8);
JpegColorConverterBase.GetConverter(JpegColorSpace.Cmyk, 8);
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="CmykColorConversion"/> class. /// Initializes a new instance of the <see cref="CmykColorConversion"/> class.

3
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/GrayscaleColorConversion.cs

@ -9,8 +9,7 @@ namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
[Config(typeof(Config.Short))] [Config(typeof(Config.Short))]
public class GrayScaleColorConversion : ColorConversionBenchmark public class GrayScaleColorConversion : ColorConversionBenchmark
{ {
private readonly JpegColorConverterBase converter = private readonly JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(JpegColorSpace.Grayscale, 8);
JpegColorConverterBase.GetConverter(JpegColorSpace.Grayscale, 8);
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="GrayScaleColorConversion"/> class. /// Initializes a new instance of the <see cref="GrayScaleColorConversion"/> class.

178
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/JpegColorPacking.cs

@ -0,0 +1,178 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using SixLabors.ImageSharp.Formats.Jpeg.Components;
namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
/// <summary>
/// Compares the previous scalar JPEG packing loops with the SIMD register-transpose implementation.
/// </summary>
[Config(typeof(Config.Short))]
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
[CategoriesColumn]
public class JpegColorPacking
{
private const float MaximumValue = 255F;
private const float Scale = 1F / MaximumValue;
private float[] x = null!;
private float[] y = null!;
private float[] z = null!;
private float[] w = null!;
private Vector3[] packed3 = null!;
private float[] destination3 = null!;
private float[] destination4 = null!;
/// <summary>
/// Gets or sets the number of pixels transformed by each benchmark invocation.
/// </summary>
[Params(128, 1024, 4096)]
public int Length { get; set; }
/// <summary>
/// Creates deterministic source and destination buffers outside the measured operations.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.x = CreateSamples(this.Length, 1);
this.y = CreateSamples(this.Length, 2);
this.z = CreateSamples(this.Length, 3);
this.w = CreateSamples(this.Length, 4);
this.packed3 = new Vector3[this.Length];
this.destination3 = new float[this.Length * 3];
this.destination4 = new float[this.Length * 4];
for (int i = 0; i < this.packed3.Length; i++)
{
this.packed3[i] = new Vector3(this.x[i], this.y[i], this.z[i]);
}
}
/// <summary>
/// Measures the previous scalar three-plane normalization and interleave loop.
/// </summary>
/// <returns>The last destination value, keeping the writes observable.</returns>
[Benchmark(Baseline = true)]
[BenchmarkCategory("Pack3")]
public float PackedNormalizeInterleave3Scalar()
{
JpegColorPackingScalar.PackedNormalizeInterleave3(this.x, this.y, this.z, this.destination3, Scale);
return this.destination3[^1];
}
/// <summary>
/// Measures the SIMD three-plane normalization and interleave implementation.
/// </summary>
/// <returns>The last destination value, keeping the writes observable.</returns>
[Benchmark]
[BenchmarkCategory("Pack3")]
public float PackedNormalizeInterleave3Simd()
{
JpegColorConverterBase.PackedNormalizeInterleave3(this.x, this.y, this.z, this.destination3, Scale);
return this.destination3[^1];
}
/// <summary>
/// Measures the previous scalar packed-three-channel deinterleave loop.
/// </summary>
/// <returns>A checksum containing the last value written to every destination plane.</returns>
[Benchmark(Baseline = true)]
[BenchmarkCategory("Unpack3")]
public float UnpackDeinterleave3Scalar()
{
JpegColorPackingScalar.UnpackDeinterleave3(this.packed3, this.x, this.y, this.z);
return this.x[^1] + this.y[^1] + this.z[^1];
}
/// <summary>
/// Measures the SIMD packed-three-channel deinterleave implementation.
/// </summary>
/// <returns>A checksum containing the last value written to every destination plane.</returns>
[Benchmark]
[BenchmarkCategory("Unpack3")]
public float UnpackDeinterleave3Simd()
{
JpegColorConverterBase.UnpackDeinterleave3(this.packed3, this.x, this.y, this.z);
return this.x[^1] + this.y[^1] + this.z[^1];
}
/// <summary>
/// Measures the previous scalar four-plane normalization and interleave loop.
/// </summary>
/// <returns>The last destination value, keeping the writes observable.</returns>
[Benchmark(Baseline = true)]
[BenchmarkCategory("Pack4")]
public float PackedNormalizeInterleave4Scalar()
{
JpegColorPackingScalar.PackedNormalizeInterleave4(this.x, this.y, this.z, this.w, this.destination4, MaximumValue);
return this.destination4[^1];
}
/// <summary>
/// Measures the SIMD four-plane normalization and interleave implementation.
/// </summary>
/// <returns>The last destination value, keeping the writes observable.</returns>
[Benchmark]
[BenchmarkCategory("Pack4")]
public float PackedNormalizeInterleave4Simd()
{
JpegColorConverterBase.PackedNormalizeInterleave4(this.x, this.y, this.z, this.w, this.destination4, MaximumValue);
return this.destination4[^1];
}
/// <summary>
/// Measures the previous scalar inverted four-plane normalization and interleave loop.
/// </summary>
/// <returns>The last destination value, keeping the writes observable.</returns>
[Benchmark(Baseline = true)]
[BenchmarkCategory("InvertPack4")]
public float PackedInvertNormalizeInterleave4Scalar()
{
JpegColorPackingScalar.PackedInvertNormalizeInterleave4(this.x, this.y, this.z, this.w, this.destination4, MaximumValue);
return this.destination4[^1];
}
/// <summary>
/// Measures the SIMD inverted four-plane normalization and interleave implementation.
/// </summary>
/// <returns>The last destination value, keeping the writes observable.</returns>
[Benchmark]
[BenchmarkCategory("InvertPack4")]
public float PackedInvertNormalizeInterleave4Simd()
{
JpegColorConverterBase.PackedInvertNormalizeInterleave4(this.x, this.y, this.z, this.w, this.destination4, MaximumValue);
return this.destination4[^1];
}
/// <summary>
/// Creates deterministic, non-integral samples for one component plane.
/// </summary>
/// <param name="length">The number of samples to create.</param>
/// <param name="component">The one-based component number used to distinguish the plane.</param>
/// <returns>The generated samples.</returns>
private static float[] CreateSamples(int length, int component)
{
float[] samples = new float[length];
for (int i = 0; i < samples.Length; i++)
{
samples[i] = (((i * 37) + (component * 53)) % 251) + (component * 0.125F);
}
return samples;
}
}

117
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/JpegColorPackingScalar.cs

@ -0,0 +1,117 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
/// <summary>
/// Preserves the scalar JPEG packing loops that preceded the SIMD investigation.
/// </summary>
internal static class JpegColorPackingScalar
{
/// <summary>
/// Normalizes and interleaves three planar component lanes using the previous scalar implementation.
/// </summary>
/// <param name="xLane">The planar X components.</param>
/// <param name="yLane">The planar Y components.</param>
/// <param name="zLane">The planar Z components.</param>
/// <param name="packed">The destination ordered as consecutive XYZ triples.</param>
/// <param name="scale">The normalization factor applied to every component.</param>
public static void PackedNormalizeInterleave3(ReadOnlySpan<float> xLane, ReadOnlySpan<float> yLane, ReadOnlySpan<float> zLane, Span<float> packed, float scale)
{
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
for (nuint i = 0; i < (nuint)xLane.Length; i++)
{
nuint packedOffset = i * 3;
Unsafe.Add(ref packedRef, packedOffset) = Unsafe.Add(ref xLaneRef, i) * scale;
Unsafe.Add(ref packedRef, packedOffset + 1) = Unsafe.Add(ref yLaneRef, i) * scale;
Unsafe.Add(ref packedRef, packedOffset + 2) = Unsafe.Add(ref zLaneRef, i) * scale;
}
}
/// <summary>
/// Deinterleaves packed XYZ values using the previous scalar implementation.
/// </summary>
/// <param name="packed">The source ordered as consecutive XYZ triples.</param>
/// <param name="xLane">The destination X components.</param>
/// <param name="yLane">The destination Y components.</param>
/// <param name="zLane">The destination Z components.</param>
public static void UnpackDeinterleave3(ReadOnlySpan<Vector3> packed, Span<float> xLane, Span<float> yLane, Span<float> zLane)
{
ref float packedRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<Vector3, float>(packed));
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
for (nuint i = 0; i < (nuint)packed.Length; i++)
{
nuint packedOffset = i * 3;
Unsafe.Add(ref xLaneRef, i) = Unsafe.Add(ref packedRef, packedOffset);
Unsafe.Add(ref yLaneRef, i) = Unsafe.Add(ref packedRef, packedOffset + 1);
Unsafe.Add(ref zLaneRef, i) = Unsafe.Add(ref packedRef, packedOffset + 2);
}
}
/// <summary>
/// Normalizes and interleaves four planar component lanes using the previous scalar implementation.
/// </summary>
/// <param name="xLane">The planar X components.</param>
/// <param name="yLane">The planar Y components.</param>
/// <param name="zLane">The planar Z components.</param>
/// <param name="wLane">The planar W components.</param>
/// <param name="packed">The destination ordered as consecutive XYZW groups.</param>
/// <param name="maximumValue">The maximum component value used to normalize each component.</param>
public static void PackedNormalizeInterleave4(ReadOnlySpan<float> xLane, ReadOnlySpan<float> yLane, ReadOnlySpan<float> zLane, ReadOnlySpan<float> wLane, Span<float> packed, float maximumValue)
{
float scale = 1F / maximumValue;
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float wLaneRef = ref MemoryMarshal.GetReference(wLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
for (nuint i = 0; i < (nuint)xLane.Length; i++)
{
nuint packedOffset = i * 4;
Unsafe.Add(ref packedRef, packedOffset) = Unsafe.Add(ref xLaneRef, i) * scale;
Unsafe.Add(ref packedRef, packedOffset + 1) = Unsafe.Add(ref yLaneRef, i) * scale;
Unsafe.Add(ref packedRef, packedOffset + 2) = Unsafe.Add(ref zLaneRef, i) * scale;
Unsafe.Add(ref packedRef, packedOffset + 3) = Unsafe.Add(ref wLaneRef, i) * scale;
}
}
/// <summary>
/// Inverts, normalizes, and interleaves four planar lanes using the previous scalar implementation.
/// </summary>
/// <param name="xLane">The inverted planar X components.</param>
/// <param name="yLane">The inverted planar Y components.</param>
/// <param name="zLane">The inverted planar Z components.</param>
/// <param name="wLane">The inverted planar W components.</param>
/// <param name="packed">The destination ordered as consecutive conventional XYZW groups.</param>
/// <param name="maximumValue">The maximum component value used for inversion and normalization.</param>
public static void PackedInvertNormalizeInterleave4(ReadOnlySpan<float> xLane, ReadOnlySpan<float> yLane, ReadOnlySpan<float> zLane, ReadOnlySpan<float> wLane, Span<float> packed, float maximumValue)
{
float scale = 1F / maximumValue;
ref float xLaneRef = ref MemoryMarshal.GetReference(xLane);
ref float yLaneRef = ref MemoryMarshal.GetReference(yLane);
ref float zLaneRef = ref MemoryMarshal.GetReference(zLane);
ref float wLaneRef = ref MemoryMarshal.GetReference(wLane);
ref float packedRef = ref MemoryMarshal.GetReference(packed);
for (nuint i = 0; i < (nuint)xLane.Length; i++)
{
nuint packedOffset = i * 4;
Unsafe.Add(ref packedRef, packedOffset) = (maximumValue - Unsafe.Add(ref xLaneRef, i)) * scale;
Unsafe.Add(ref packedRef, packedOffset + 1) = (maximumValue - Unsafe.Add(ref yLaneRef, i)) * scale;
Unsafe.Add(ref packedRef, packedOffset + 2) = (maximumValue - Unsafe.Add(ref zLaneRef, i)) * scale;
Unsafe.Add(ref packedRef, packedOffset + 3) = (maximumValue - Unsafe.Add(ref wLaneRef, i)) * scale;
}
}
}

3
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/RgbColorConversion.cs

@ -9,8 +9,7 @@ namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
[Config(typeof(Config.Short))] [Config(typeof(Config.Short))]
public class RgbColorConversion : ColorConversionBenchmark public class RgbColorConversion : ColorConversionBenchmark
{ {
private readonly JpegColorConverterBase converter = private readonly JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(JpegColorSpace.RGB, 8);
JpegColorConverterBase.GetConverter(JpegColorSpace.RGB, 8);
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="RgbColorConversion"/> class. /// Initializes a new instance of the <see cref="RgbColorConversion"/> class.

3
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/YCbCrColorConversion.cs

@ -9,8 +9,7 @@ namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
[Config(typeof(Config.Short))] [Config(typeof(Config.Short))]
public class YCbCrColorConversion : ColorConversionBenchmark public class YCbCrColorConversion : ColorConversionBenchmark
{ {
private readonly JpegColorConverterBase converter = private readonly JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(JpegColorSpace.YCbCr, 8);
JpegColorConverterBase.GetConverter(JpegColorSpace.YCbCr, 8);
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="YCbCrColorConversion"/> class. /// Initializes a new instance of the <see cref="YCbCrColorConversion"/> class.

244
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/YCbCrOperatorAssembly.cs

@ -1,244 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Runtime.Intrinsics;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using SixLabors.ImageSharp.Formats.Jpeg.Components;
namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
/// <summary>
/// Exposes every YCbCr operator overload directly to the disassembly diagnoser.
/// </summary>
[Config(typeof(Config.Analysis))]
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
[CategoriesColumn]
public class YCbCrOperatorAssembly
{
private const float MaximumValue = 255F;
private const float HalfValue = 128F;
private const float Scale = 1F / MaximumValue;
private float scalarC0 = 64F;
private float scalarC1 = 96F;
private float scalarC2 = 160F;
private readonly Vector128<float> vector128C0 = Vector128.Create(64F);
private readonly Vector128<float> vector128C1 = Vector128.Create(96F);
private readonly Vector128<float> vector128C2 = Vector128.Create(160F);
private readonly Vector128<float> vector128Maximum = Vector128.Create(MaximumValue);
private readonly Vector128<float> vector128Half = Vector128.Create(HalfValue);
private readonly Vector128<float> vector128Scale = Vector128.Create(Scale);
private readonly Vector256<float> vector256C0 = Vector256.Create(64F);
private readonly Vector256<float> vector256C1 = Vector256.Create(96F);
private readonly Vector256<float> vector256C2 = Vector256.Create(160F);
private readonly Vector256<float> vector256Maximum = Vector256.Create(MaximumValue);
private readonly Vector256<float> vector256Half = Vector256.Create(HalfValue);
private readonly Vector256<float> vector256Scale = Vector256.Create(Scale);
private readonly Vector512<float> vector512C0 = Vector512.Create(64F);
private readonly Vector512<float> vector512C1 = Vector512.Create(96F);
private readonly Vector512<float> vector512C2 = Vector512.Create(160F);
private readonly Vector512<float> vector512Maximum = Vector512.Create(MaximumValue);
private readonly Vector512<float> vector512Half = Vector512.Create(HalfValue);
private readonly Vector512<float> vector512Scale = Vector512.Create(Scale);
/// <summary>
/// Invokes the scalar JPEG-to-RGB operator.
/// </summary>
/// <returns>A checksum containing all three converted channels.</returns>
[Benchmark]
[BenchmarkCategory("ToRgb")]
public float ToRgbScalar()
{
float c0 = this.scalarC0;
float c1 = this.scalarC1;
float c2 = this.scalarC2;
JpegColorConverterBase.YCbCrOperator.ConvertToRgb(
ref c0,
ref c1,
ref c2,
0,
MaximumValue,
HalfValue,
Scale);
// Returning the channel sum keeps every output live in the generated assembly.
return c0 + c1 + c2;
}
/// <summary>
/// Invokes the Vector128 JPEG-to-RGB operator.
/// </summary>
/// <returns>A checksum containing all three converted channel vectors.</returns>
[Benchmark]
[BenchmarkCategory("ToRgb")]
public Vector128<float> ToRgbVector128()
{
Vector128<float> c0 = this.vector128C0;
Vector128<float> c1 = this.vector128C1;
Vector128<float> c2 = this.vector128C2;
JpegColorConverterBase.YCbCrOperator.ConvertToRgb(
ref c0,
ref c1,
ref c2,
default,
this.vector128Maximum,
this.vector128Half,
this.vector128Scale);
// The vector sum makes all RGB results observable without adding stores to the measured body.
return c0 + c1 + c2;
}
/// <summary>
/// Invokes the Vector256 JPEG-to-RGB operator.
/// </summary>
/// <returns>A checksum containing all three converted channel vectors.</returns>
[Benchmark]
[BenchmarkCategory("ToRgb")]
public Vector256<float> ToRgbVector256()
{
Vector256<float> c0 = this.vector256C0;
Vector256<float> c1 = this.vector256C1;
Vector256<float> c2 = this.vector256C2;
JpegColorConverterBase.YCbCrOperator.ConvertToRgb(
ref c0,
ref c1,
ref c2,
default,
this.vector256Maximum,
this.vector256Half,
this.vector256Scale);
// The vector sum makes all RGB results observable without adding stores to the measured body.
return c0 + c1 + c2;
}
/// <summary>
/// Invokes the Vector512 JPEG-to-RGB operator.
/// </summary>
/// <returns>A checksum containing all three converted channel vectors.</returns>
[Benchmark]
[BenchmarkCategory("ToRgb")]
public Vector512<float> ToRgbVector512()
{
Vector512<float> c0 = this.vector512C0;
Vector512<float> c1 = this.vector512C1;
Vector512<float> c2 = this.vector512C2;
JpegColorConverterBase.YCbCrOperator.ConvertToRgb(
ref c0,
ref c1,
ref c2,
default,
this.vector512Maximum,
this.vector512Half,
this.vector512Scale);
// The vector sum makes all RGB results observable without adding stores to the measured body.
return c0 + c1 + c2;
}
/// <summary>
/// Invokes the scalar RGB-to-JPEG operator.
/// </summary>
/// <returns>A checksum containing all converted components.</returns>
[Benchmark]
[BenchmarkCategory("FromRgb")]
public float FromRgbScalar()
{
JpegColorConverterBase.YCbCrOperator.ConvertFromRgb(
this.scalarC0,
this.scalarC1,
this.scalarC2,
MaximumValue,
HalfValue,
Scale,
out float c0,
out float c1,
out float c2,
out float c3);
// c3 is deliberately included so a future four-component implementation remains observable.
return c0 + c1 + c2 + c3;
}
/// <summary>
/// Invokes the Vector128 RGB-to-JPEG operator.
/// </summary>
/// <returns>A checksum containing all converted component vectors.</returns>
[Benchmark]
[BenchmarkCategory("FromRgb")]
public Vector128<float> FromRgbVector128()
{
JpegColorConverterBase.YCbCrOperator.ConvertFromRgb(
this.vector128C0,
this.vector128C1,
this.vector128C2,
this.vector128Maximum,
this.vector128Half,
this.vector128Scale,
out Vector128<float> c0,
out Vector128<float> c1,
out Vector128<float> c2,
out Vector128<float> c3);
// Include all planar results in the returned vector so the JIT retains every calculation.
return c0 + c1 + c2 + c3;
}
/// <summary>
/// Invokes the Vector256 RGB-to-JPEG operator.
/// </summary>
/// <returns>A checksum containing all converted component vectors.</returns>
[Benchmark]
[BenchmarkCategory("FromRgb")]
public Vector256<float> FromRgbVector256()
{
JpegColorConverterBase.YCbCrOperator.ConvertFromRgb(
this.vector256C0,
this.vector256C1,
this.vector256C2,
this.vector256Maximum,
this.vector256Half,
this.vector256Scale,
out Vector256<float> c0,
out Vector256<float> c1,
out Vector256<float> c2,
out Vector256<float> c3);
// Include all planar results in the returned vector so the JIT retains every calculation.
return c0 + c1 + c2 + c3;
}
/// <summary>
/// Invokes the Vector512 RGB-to-JPEG operator.
/// </summary>
/// <returns>A checksum containing all converted component vectors.</returns>
[Benchmark]
[BenchmarkCategory("FromRgb")]
public Vector512<float> FromRgbVector512()
{
JpegColorConverterBase.YCbCrOperator.ConvertFromRgb(
this.vector512C0,
this.vector512C1,
this.vector512C2,
this.vector512Maximum,
this.vector512Half,
this.vector512Scale,
out Vector512<float> c0,
out Vector512<float> c1,
out Vector512<float> c2,
out Vector512<float> c3);
// Include all planar results in the returned vector so the JIT retains every calculation.
return c0 + c1 + c2 + c3;
}
}

3
tests/ImageSharp.Benchmarks/Codecs/Jpeg/ColorConversion/YccKColorConverter.cs

@ -9,8 +9,7 @@ namespace SixLabors.ImageSharp.Benchmarks.Codecs.Jpeg;
[Config(typeof(Config.Short))] [Config(typeof(Config.Short))]
public class YccKColorConverter : ColorConversionBenchmark public class YccKColorConverter : ColorConversionBenchmark
{ {
private readonly JpegColorConverterBase converter = private readonly JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(JpegColorSpace.Ycck, 8);
JpegColorConverterBase.GetConverter(JpegColorSpace.Ycck, 8);
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="YccKColorConverter"/> class. /// Initializes a new instance of the <see cref="YccKColorConverter"/> class.

64
tests/ImageSharp.Benchmarks/Codecs/Png/PngFilterEncodeAssembly.cs

@ -1,64 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.Formats.Png.Filters;
namespace SixLabors.ImageSharp.Benchmarks.Codecs.Png;
/// <summary>
/// Exposes every normalized PNG filter for assembly inspection.
/// </summary>
[Config(typeof(Config.Analysis))]
public class PngFilterEncodeAssembly
{
private const int BytesPerPixel = 4;
private const int Count = 180;
private byte[] scanline;
private byte[] previousScanline;
private byte[] result;
/// <summary>
/// Creates inputs whose suffix exercises 512-, 256-, and 128-bit register widths.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.scanline = new byte[Count];
this.previousScanline = new byte[Count];
this.result = new byte[Count + 1];
Random random = new(12345678);
random.NextBytes(this.scanline);
random.NextBytes(this.previousScanline);
}
/// <summary>
/// Executes the normalized Sub encoder.
/// </summary>
[Benchmark]
public void Sub()
=> SubFilter.Encode(this.scanline, this.result, BytesPerPixel, out _);
/// <summary>
/// Executes the normalized Up encoder.
/// </summary>
[Benchmark]
public void Up()
=> UpFilter.Encode(this.scanline, this.previousScanline, this.result, out _);
/// <summary>
/// Executes the normalized Average encoder.
/// </summary>
[Benchmark]
public void Average()
=> AverageFilter.Encode(this.scanline, this.previousScanline, this.result, BytesPerPixel, out _);
/// <summary>
/// Executes the normalized Paeth encoder.
/// </summary>
[Benchmark]
public void Paeth()
=> PaethFilter.Encode(this.scanline, this.previousScanline, this.result, BytesPerPixel, out _);
}

175
tests/ImageSharp.Benchmarks/General/BasicMath/TensorPrimitivesAssembly.cs

@ -1,175 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Benchmarks.General.BasicMath;
/// <summary>
/// Exposes every floating-point tensor compatibility operation for assembly inspection.
/// </summary>
[Config(typeof(Config.Analysis))]
public class TensorPrimitivesAssembly
{
private const int Count = 2048;
private readonly float[] x = new float[Count];
private readonly float[] y = new float[Count];
private readonly float[] destination = new float[Count];
/// <summary>
/// Populates the input spans with deterministic non-uniform values.
/// </summary>
[GlobalSetup]
public void Setup()
{
for (int i = 0; i < Count; i++)
{
this.x[i] = ((i * 17) % 251) + 1;
this.y[i] = ((i * 29) % 251) + 1;
}
}
/// <summary>
/// Adds two floating-point spans.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public float Add()
{
TensorPrimitives_.Add<float>(this.x, this.y, this.destination);
return this.destination[0];
}
/// <summary>
/// Clamps a floating-point span between scalar bounds.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public float Clamp()
{
TensorPrimitives_.Clamp(this.x, 64F, 128F, this.destination);
return this.destination[0];
}
/// <summary>
/// Divides a floating-point span by a scalar.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public float Divide()
{
TensorPrimitives_.Divide(this.x, 4096F, this.destination);
return this.destination[0];
}
/// <summary>
/// Computes the element-wise maximum of a floating-point span and a scalar.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public float Max()
{
TensorPrimitives_.Max(this.x, 64F, this.destination);
return this.destination[0];
}
/// <summary>
/// Multiplies a floating-point span by a scalar.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public float Multiply()
{
TensorPrimitives_.Multiply(this.x, 0.5F, this.destination);
return this.destination[0];
}
}
/// <summary>
/// Exposes integral addition specializations for assembly inspection.
/// </summary>
/// <typeparam name="T">The integral element type.</typeparam>
[Config(typeof(Config.Analysis))]
[GenericTypeArguments(typeof(byte))]
[GenericTypeArguments(typeof(uint))]
public class TensorPrimitivesIntegralAddAssembly<T>
where T : unmanaged, INumber<T>
{
private const int Count = 2048;
private readonly T[] x = new T[Count];
private readonly T[] y = new T[Count];
private readonly T[] destination = new T[Count];
/// <summary>
/// Populates the input spans with deterministic non-uniform values.
/// </summary>
[GlobalSetup]
public void Setup()
{
for (int i = 0; i < Count; i++)
{
this.x[i] = T.CreateTruncating((i * 17) + 31);
this.y[i] = T.CreateTruncating((i * 29) + 7);
}
}
/// <summary>
/// Adds two integral spans.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public T Add()
{
TensorPrimitives_.Add<T>(this.x, this.y, this.destination);
return this.destination[0];
}
}
/// <summary>
/// Exposes integral clamp specializations for assembly inspection.
/// </summary>
/// <typeparam name="T">The integral element type.</typeparam>
[Config(typeof(Config.Analysis))]
[GenericTypeArguments(typeof(byte))]
[GenericTypeArguments(typeof(uint))]
[GenericTypeArguments(typeof(int))]
public class TensorPrimitivesIntegralClampAssembly<T>
where T : unmanaged, INumber<T>
{
private const int Count = 2048;
private readonly T[] source = new T[Count];
private readonly T[] destination = new T[Count];
private T min;
private T max;
/// <summary>
/// Populates the input span and scalar bounds with deterministic values.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.min = T.CreateTruncating(64);
this.max = T.CreateTruncating(128);
for (int i = 0; i < Count; i++)
{
this.source[i] = T.CreateTruncating((i * 31) % 257);
}
}
/// <summary>
/// Clamps an integral span between scalar bounds.
/// </summary>
/// <returns>The first result, which keeps the destination observable.</returns>
[Benchmark]
public T Clamp()
{
TensorPrimitives_.Clamp(this.source, this.min, this.max, this.destination);
return this.destination[0];
}
}

128
tests/ImageSharp.Benchmarks/General/PixelConversion/PackedPixelConversionAssembly.cs

@ -1,128 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using BenchmarkDotNet.Attributes;
namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion;
/// <summary>
/// Exposes every stateless packed-pixel shuffle operator for assembly inspection.
/// </summary>
/// <remarks>
/// Seven pixels expose the short-input and vector-tail paths. Seventeen pixels execute one
/// full intrinsic group and leave one complete pixel for the scalar remainder, making every
/// part of each generated traversal observable.
/// </remarks>
[Config(typeof(Config.Analysis))]
public class PackedPixelConversionAssembly
{
private byte[] source3;
private byte[] source4;
private byte[] destination3;
private byte[] destination4;
/// <summary>
/// Gets or sets the number of pixels converted by each invocation.
/// </summary>
[Params(7, 17)]
public int Count { get; set; }
/// <summary>
/// Populates the source buffers with deterministic non-uniform channel values.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.source3 = new byte[this.Count * 3];
this.source4 = new byte[this.Count * 4];
this.destination3 = new byte[this.Count * 3];
this.destination4 = new byte[this.Count * 4];
new Random(42).NextBytes(this.source3);
new Random(42).NextBytes(this.source4);
}
/// <summary>
/// Executes the WXYZ four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Wxyz() => SimdUtils.Shuffle4<WXYZShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the WZYX four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Wzyx() => SimdUtils.Shuffle4<WZYXShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the YZWX four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Yzwx() => SimdUtils.Shuffle4<YZWXShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the ZYXW four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Zyxw() => SimdUtils.Shuffle4<ZYXWShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the XWZY four-to-four operator.
/// </summary>
[Benchmark]
public void Shuffle4Xwzy() => SimdUtils.Shuffle4<XWZYShuffle4>(this.source4, this.destination4);
/// <summary>
/// Executes the XYZ four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Xyz() => SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the YZW four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Yzw() => SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the WZY four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Wzy() => SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the ZYX four-to-three operator.
/// </summary>
[Benchmark]
public void Slice3Zyx() => SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(this.source4, this.destination3);
/// <summary>
/// Executes the XYZW three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Xyzw() => SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the WXYZ three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Wxyz() => SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the WZYX three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Wzyx() => SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the ZYXW three-to-four operator.
/// </summary>
[Benchmark]
public void Pad4Zyxw() => SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(this.source3, this.destination4);
/// <summary>
/// Executes the ZYX three-to-three operator.
/// </summary>
[Benchmark]
public void Shuffle3Zyx() => SimdUtils.Shuffle3<ZYXShuffle3>(this.source3, this.destination3);
}

59
tests/ImageSharp.Benchmarks/General/PixelConversion/Vector4AffineTransformAssembly.cs

@ -1,59 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.PixelFormats.Utils;
namespace SixLabors.ImageSharp.Benchmarks.General.PixelConversion;
/// <summary>
/// Exposes every stateful affine operator and traversal remainder for assembly inspection.
/// </summary>
[Config(typeof(Config.Analysis))]
public class Vector4AffineTransformAssembly
{
private static readonly Vector4 Multiplier = new(255F, 2F, 65535F, .5F);
private static readonly Vector4 Offset = new(17F, -1F, 32768F, 3F);
private static readonly Vector4 Divisor = new(255F, 2F, 65535F, .5F);
private Vector4[] vectors;
/// <summary>
/// Gets or sets the number of vectors transformed by each invocation.
/// </summary>
/// <remarks>
/// Three vectors exercise the 256- and 128-bit stages. Seventeen vectors exercise
/// the 512-bit loop and leave one vector for the 128-bit remainder.
/// </remarks>
[Params(3, 17)]
public int Count { get; set; }
/// <summary>
/// Creates a non-uniform input buffer.
/// </summary>
[GlobalSetup]
public void Setup()
{
this.vectors = new Vector4[this.Count];
for (int i = 0; i < this.vectors.Length; i++)
{
this.vectors[i] = new Vector4(i + .25F, i + .5F, i + .75F, i + 1F);
}
}
/// <summary>
/// Executes the multiply-then-add stateful operator.
/// </summary>
[Benchmark]
public void MultiplyThenAdd()
=> Vector4Converters.MultiplyThenAdd(this.vectors, Multiplier, Offset);
/// <summary>
/// Executes the add-then-divide stateful operator.
/// </summary>
[Benchmark]
public void AddThenDivide()
=> Vector4Converters.AddThenDivide(this.vectors, Offset, Divisor);
}

327
tests/ImageSharp.Benchmarks/PixelBlenders/PixelBlenderTraversalAssembly.cs

@ -1,327 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using System.Runtime.CompilerServices;
using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.PixelFormats.PixelBlenders;
namespace SixLabors.ImageSharp.Benchmarks.PixelBlenders;
/// <summary>
/// Exposes every shared pixel-blender traversal shape for assembly inspection.
/// </summary>
/// <remarks>
/// Seven pixels leave three scalar pixels after AVX-512 or one scalar pixel after AVX2, so each
/// hardware job contains both its widest supported loop and the portable Vector4 remainder.
/// </remarks>
[Config(typeof(Config.Analysis))]
public class PixelBlenderTraversalAssembly
{
private const int Count = 7;
private const float Amount = .625F;
private readonly ExposedNormalSrcOverBlender blender = new();
private readonly Vector4[] destination = new Vector4[Count];
private readonly Vector4[] background = new Vector4[Count];
private readonly Vector4[] source = new Vector4[Count];
private readonly float[] amounts = new float[Count];
private readonly float[] coverage = new float[Count];
private Vector4 constantSource;
/// <summary>
/// Populates all lanes with deterministic, non-constant values.
/// </summary>
[GlobalSetup]
public void Setup()
{
Random random = new(42);
for (int i = 0; i < Count; i++)
{
// Distinct RGBA lanes make incorrect pixel grouping visible in both results and assembly.
this.background[i] = CreatePixel(random);
this.source[i] = CreatePixel(random);
this.amounts[i] = random.NextSingle();
this.coverage[i] = random.NextSingle();
}
this.constantSource = CreatePixel(random);
}
/// <summary>
/// Blends a source row with one shared amount.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 SourceSpanScalarAmount()
{
this.blender.BlendSourceSpanScalarAmount(
this.destination,
this.background,
this.source,
Amount);
return this.destination[^1];
}
/// <summary>
/// Blends a constant source with one shared amount.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 ConstantSourceScalarAmount()
{
this.blender.BlendConstantSourceScalarAmount(
this.destination,
this.background,
this.constantSource,
Amount);
return this.destination[^1];
}
/// <summary>
/// Blends a source row with per-pixel amounts.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 SourceSpanAmountSpan()
{
this.blender.BlendSourceSpanAmountSpan(
this.destination,
this.background,
this.source,
this.amounts);
return this.destination[^1];
}
/// <summary>
/// Blends a constant source with per-pixel amounts.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 ConstantSourceAmountSpan()
{
this.blender.BlendConstantSourceAmountSpan(
this.destination,
this.background,
this.constantSource,
this.amounts);
return this.destination[^1];
}
/// <summary>
/// Blends a source row with one shared amount and per-pixel coverage.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 SourceSpanScalarAmountCoverage()
{
this.blender.BlendSourceSpanScalarAmountCoverage(
this.destination,
this.background,
this.source,
Amount,
this.coverage);
return this.destination[^1];
}
/// <summary>
/// Blends a constant source with one shared amount and per-pixel coverage.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 ConstantSourceScalarAmountCoverage()
{
this.blender.BlendConstantSourceScalarAmountCoverage(
this.destination,
this.background,
this.constantSource,
Amount,
this.coverage);
return this.destination[^1];
}
/// <summary>
/// Blends a source row with per-pixel amounts and coverage.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 SourceSpanAmountSpanCoverage()
{
this.blender.BlendSourceSpanAmountSpanCoverage(
this.destination,
this.background,
this.source,
this.amounts,
this.coverage);
return this.destination[^1];
}
/// <summary>
/// Blends a constant source with per-pixel amounts and coverage.
/// </summary>
/// <returns>The final destination pixel.</returns>
[Benchmark]
public Vector4 ConstantSourceAmountSpanCoverage()
{
this.blender.BlendConstantSourceAmountSpanCoverage(
this.destination,
this.background,
this.constantSource,
this.amounts,
this.coverage);
return this.destination[^1];
}
/// <summary>
/// Creates one non-constant RGBA sample.
/// </summary>
/// <param name="random">The deterministic value source.</param>
/// <returns>The sample pixel.</returns>
private static Vector4 CreatePixel(Random random)
=> new(random.NextSingle(), random.NextSingle(), random.NextSingle(), random.NextSingle());
/// <summary>
/// Exposes the protected shared traversal overloads without adding benchmark hooks to production APIs.
/// </summary>
private sealed class ExposedNormalSrcOverBlender :
DefaultPixelBlender<RgbaVector, DefaultPixelBlenderOperators.NormalSrcOver>
{
/// <summary>
/// Invokes the source-span, scalar-amount traversal.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The source vectors.</param>
/// <param name="amount">The shared source amount.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendSourceSpanScalarAmount(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
float amount)
=> this.BlendFunction(destination, background, source, amount);
/// <summary>
/// Invokes the constant-source, scalar-amount traversal.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The constant source vector.</param>
/// <param name="amount">The shared source amount.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendConstantSourceScalarAmount(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
float amount)
=> this.BlendFunction(destination, background, source, amount);
/// <summary>
/// Invokes the source-span, amount-span traversal.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The source vectors.</param>
/// <param name="amount">The per-pixel source amounts.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendSourceSpanAmountSpan(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
ReadOnlySpan<float> amount)
=> this.BlendFunction(destination, background, source, amount);
/// <summary>
/// Invokes the constant-source, amount-span traversal.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The constant source vector.</param>
/// <param name="amount">The per-pixel source amounts.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendConstantSourceAmountSpan(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
ReadOnlySpan<float> amount)
=> this.BlendFunction(destination, background, source, amount);
/// <summary>
/// Invokes the source-span, scalar-amount traversal with coverage.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The source vectors.</param>
/// <param name="amount">The shared source amount.</param>
/// <param name="coverage">The per-pixel coverage values.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendSourceSpanScalarAmountCoverage(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
float amount,
ReadOnlySpan<float> coverage)
=> this.BlendWithCoverageFunction(destination, background, source, amount, coverage);
/// <summary>
/// Invokes the constant-source, scalar-amount traversal with coverage.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The constant source vector.</param>
/// <param name="amount">The shared source amount.</param>
/// <param name="coverage">The per-pixel coverage values.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendConstantSourceScalarAmountCoverage(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
float amount,
ReadOnlySpan<float> coverage)
=> this.BlendWithCoverageFunction(destination, background, source, amount, coverage);
/// <summary>
/// Invokes the source-span, amount-span traversal with coverage.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The source vectors.</param>
/// <param name="amount">The per-pixel source amounts.</param>
/// <param name="coverage">The per-pixel coverage values.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendSourceSpanAmountSpanCoverage(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
ReadOnlySpan<Vector4> source,
ReadOnlySpan<float> amount,
ReadOnlySpan<float> coverage)
=> this.BlendWithCoverageFunction(destination, background, source, amount, coverage);
/// <summary>
/// Invokes the constant-source, amount-span traversal with coverage.
/// </summary>
/// <param name="destination">The destination vectors.</param>
/// <param name="background">The background vectors.</param>
/// <param name="source">The constant source vector.</param>
/// <param name="amount">The per-pixel source amounts.</param>
/// <param name="coverage">The per-pixel coverage values.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void BlendConstantSourceAmountSpanCoverage(
Span<Vector4> destination,
ReadOnlySpan<Vector4> background,
Vector4 source,
ReadOnlySpan<float> amount,
ReadOnlySpan<float> coverage)
=> this.BlendWithCoverageFunction(destination, background, source, amount, coverage);
}
}

70
tests/ImageSharp.Tests/Common/SimdUtilsTests.Shuffle.cs

@ -303,30 +303,15 @@ public partial class SimdUtilsTests
{ {
int size = FeatureTestRunner.Deserialize<int>(serialized); int size = FeatureTestRunner.Deserialize<int>(serialized);
TestShuffleByte4Channel( TestShuffleByte4Channel(size, (s, d) => SimdUtils.Shuffle4<WXYZShuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle2103);
size,
(s, d) => SimdUtils.Shuffle4<WXYZShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle2103);
TestShuffleByte4Channel( TestShuffleByte4Channel(size, (s, d) => SimdUtils.Shuffle4<WZYXShuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0123);
size,
(s, d) => SimdUtils.Shuffle4<WZYXShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0123);
TestShuffleByte4Channel( TestShuffleByte4Channel(size, (s, d) => SimdUtils.Shuffle4<YZWXShuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0321);
size,
(s, d) => SimdUtils.Shuffle4<YZWXShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0321);
TestShuffleByte4Channel( TestShuffleByte4Channel(size, (s, d) => SimdUtils.Shuffle4<ZYXWShuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3012);
size,
(s, d) => SimdUtils.Shuffle4<ZYXWShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
TestShuffleByte4Channel( TestShuffleByte4Channel(size, (s, d) => SimdUtils.Shuffle4<XWZYShuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle1230);
size,
(s, d) => SimdUtils.Shuffle4<XWZYShuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle1230);
} }
FeatureTestRunner.RunWithHwIntrinsicsFeature( FeatureTestRunner.RunWithHwIntrinsicsFeature(
@ -343,10 +328,7 @@ public partial class SimdUtilsTests
{ {
int size = FeatureTestRunner.Deserialize<int>(serialized); int size = FeatureTestRunner.Deserialize<int>(serialized);
TestShuffleByte3Channel( TestShuffleByte3Channel(size, (s, d) => SimdUtils.Shuffle3<ZYXShuffle3>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3012);
size,
(s, d) => SimdUtils.Shuffle3<ZYXShuffle3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
} }
FeatureTestRunner.RunWithHwIntrinsicsFeature( FeatureTestRunner.RunWithHwIntrinsicsFeature(
@ -363,25 +345,13 @@ public partial class SimdUtilsTests
{ {
int size = FeatureTestRunner.Deserialize<int>(serialized); int size = FeatureTestRunner.Deserialize<int>(serialized);
TestPad3Shuffle4Channel( TestPad3Shuffle4Channel(size, (s, d) => SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3210);
size,
(s, d) => SimdUtils.Pad3Shuffle4<XYZWPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3210);
TestPad3Shuffle4Channel( TestPad3Shuffle4Channel(size, (s, d) => SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle2103);
size,
(s, d) => SimdUtils.Pad3Shuffle4<WXYZPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle2103);
TestPad3Shuffle4Channel( TestPad3Shuffle4Channel(size, (s, d) => SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0123);
size,
(s, d) => SimdUtils.Pad3Shuffle4<WZYXPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0123);
TestPad3Shuffle4Channel( TestPad3Shuffle4Channel(size, (s, d) => SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3012);
size,
(s, d) => SimdUtils.Pad3Shuffle4<ZYXWPad3Shuffle4>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
} }
FeatureTestRunner.RunWithHwIntrinsicsFeature( FeatureTestRunner.RunWithHwIntrinsicsFeature(
@ -398,25 +368,13 @@ public partial class SimdUtilsTests
{ {
int size = FeatureTestRunner.Deserialize<int>(serialized); int size = FeatureTestRunner.Deserialize<int>(serialized);
TestShuffle4Slice3Channel( TestShuffle4Slice3Channel(size, (s, d) => SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3210);
size,
(s, d) => SimdUtils.Shuffle4Slice3<XYZWShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3210);
TestShuffle4Slice3Channel( TestShuffle4Slice3Channel(size, (s, d) => SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0321);
size,
(s, d) => SimdUtils.Shuffle4Slice3<YZWXShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0321);
TestShuffle4Slice3Channel( TestShuffle4Slice3Channel(size, (s, d) => SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle0123);
size,
(s, d) => SimdUtils.Shuffle4Slice3<WZYXShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle0123);
TestShuffle4Slice3Channel( TestShuffle4Slice3Channel(size, (s, d) => SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(s.Span, d.Span), SimdUtils.Shuffle.MMShuffle3012);
size,
(s, d) => SimdUtils.Shuffle4Slice3<ZYXWShuffle4Slice3>(s.Span, d.Span),
SimdUtils.Shuffle.MMShuffle3012);
} }
FeatureTestRunner.RunWithHwIntrinsicsFeature( FeatureTestRunner.RunWithHwIntrinsicsFeature(

105
tests/ImageSharp.Tests/Formats/Jpg/JpegColorConverterTests.cs

@ -17,8 +17,7 @@ public class JpegColorConverterTests
private const float FromRgbTolerance = 0.01F; private const float FromRgbTolerance = 0.01F;
// Independent model checks compare normalized colors at one tenth of a byte-domain sample. // Independent model checks compare normalized colors at one tenth of a byte-domain sample.
private static readonly ApproximateColorProfileComparer ColorSpaceComparer = private static readonly ApproximateColorProfileComparer ColorSpaceComparer = new(epsilon: ColorProfileTolerance);
new(epsilon: ColorProfileTolerance);
/// <summary> /// <summary>
/// Verifies that unsupported color spaces are rejected by the converter factory. /// Verifies that unsupported color spaces are rejected by the converter factory.
@ -77,27 +76,13 @@ public class JpegColorConverterTests
/// <param name="colorSpace">The JPEG color space.</param> /// <param name="colorSpace">The JPEG color space.</param>
/// <param name="expectedType">The expected closed converter type.</param> /// <param name="expectedType">The expected closed converter type.</param>
[Theory] [Theory]
[InlineData( [InlineData(JpegColorSpace.Grayscale, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.GrayScaleOperator>))]
JpegColorSpace.Grayscale, [InlineData(JpegColorSpace.RGB, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.RgbOperator>))]
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.GrayScaleOperator>))] [InlineData(JpegColorSpace.Cmyk, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.CmykOperator>))]
[InlineData( [InlineData(JpegColorSpace.YCbCr, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.YCbCrOperator>))]
JpegColorSpace.RGB, [InlineData(JpegColorSpace.Ycck, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.YccKOperator>))]
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.RgbOperator>))] [InlineData(JpegColorSpace.TiffCmyk, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.TiffCmykOperator>))]
[InlineData( [InlineData(JpegColorSpace.TiffYccK, typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.TiffYccKOperator>))]
JpegColorSpace.Cmyk,
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.CmykOperator>))]
[InlineData(
JpegColorSpace.YCbCr,
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.YCbCrOperator>))]
[InlineData(
JpegColorSpace.Ycck,
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.YccKOperator>))]
[InlineData(
JpegColorSpace.TiffCmyk,
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.TiffCmykOperator>))]
[InlineData(
JpegColorSpace.TiffYccK,
typeof(JpegColorConverterBase.JpegColorConverter<JpegColorConverterBase.TiffYccKOperator>))]
internal void GetConverterReturnsClosedOperatorConverter(JpegColorSpace colorSpace, Type expectedType) internal void GetConverterReturnsClosedOperatorConverter(JpegColorSpace colorSpace, Type expectedType)
{ {
JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(colorSpace, 8); JpegColorConverterBase converter = JpegColorConverterBase.GetConverter(colorSpace, 8);
@ -152,10 +137,7 @@ public class JpegColorConverterTests
[InlineData(JpegColorSpace.TiffCmyk, 4, 12)] [InlineData(JpegColorSpace.TiffCmyk, 4, 12)]
[InlineData(JpegColorSpace.TiffYccK, 4, 8)] [InlineData(JpegColorSpace.TiffYccK, 4, 8)]
[InlineData(JpegColorSpace.TiffYccK, 4, 12)] [InlineData(JpegColorSpace.TiffYccK, 4, 12)]
internal void OperatorTraversalMatchesScalarDefinition( internal void OperatorTraversalMatchesScalarDefinition(JpegColorSpace colorSpace, int componentCount, int precision)
JpegColorSpace colorSpace,
int componentCount,
int precision)
{ {
switch (colorSpace) switch (colorSpace)
{ {
@ -191,9 +173,7 @@ public class JpegColorConverterTests
/// </summary> /// </summary>
[Fact] [Fact]
public void OperatorTraversalMatchesScalarWithoutHardwareIntrinsics() public void OperatorTraversalMatchesScalarWithoutHardwareIntrinsics()
=> FeatureTestRunner.RunWithHwIntrinsicsFeature( => FeatureTestRunner.RunWithHwIntrinsicsFeature(RunWithoutHardwareIntrinsics, HwIntrinsics.DisableHWIntrinsic);
RunWithoutHardwareIntrinsics,
HwIntrinsics.DisableHWIntrinsic);
/// <summary> /// <summary>
/// Verifies TIFF YccK encoding against the canonical normalized color-profile conversion. /// Verifies TIFF YccK encoding against the canonical normalized color-profile conversion.
@ -264,8 +244,7 @@ public class JpegColorConverterTests
private static void ValidateOperator<TOperator>(int componentCount, int precision) private static void ValidateOperator<TOperator>(int componentCount, int precision)
where TOperator : struct, JpegColorConverterBase.IJpegColorConverterOperator where TOperator : struct, JpegColorConverterBase.IJpegColorConverterOperator
{ {
JpegColorConverterBase converter = JpegColorConverterBase converter = new JpegColorConverterBase.JpegColorConverter<TOperator>(precision);
new JpegColorConverterBase.JpegColorConverter<TOperator>(precision);
int[] lengths = [1, 3, 4, 7, 8, 15, 16, 31, 32, 40, 64, 128]; int[] lengths = [1, 3, 4, 7, 8, 15, 16, 31, 32, 40, 64, 128];
// Adjacent values around 4/8/16 lanes verify every prefix, mixed-width tail, and scalar remainder. // Adjacent values around 4/8/16 lanes verify every prefix, mixed-width tail, and scalar remainder.
@ -284,11 +263,7 @@ public class JpegColorConverterTests
/// <param name="length">The number of samples to convert.</param> /// <param name="length">The number of samples to convert.</param>
/// <param name="componentCount">The number of source component planes.</param> /// <param name="componentCount">The number of source component planes.</param>
/// <param name="precision">The JPEG sample precision.</param> /// <param name="precision">The JPEG sample precision.</param>
private static void ValidateConversionToRgb<TOperator>( private static void ValidateConversionToRgb<TOperator>(JpegColorConverterBase converter, int length, int componentCount, int precision)
JpegColorConverterBase converter,
int length,
int componentCount,
int precision)
where TOperator : struct, JpegColorConverterBase.IJpegColorConverterOperator where TOperator : struct, JpegColorConverterBase.IJpegColorConverterOperator
{ {
JpegColorConverterBase.ComponentValues expected = CreateRandomValues(length, componentCount, precision); JpegColorConverterBase.ComponentValues expected = CreateRandomValues(length, componentCount, precision);
@ -326,11 +301,7 @@ public class JpegColorConverterTests
/// <param name="length">The number of samples to convert.</param> /// <param name="length">The number of samples to convert.</param>
/// <param name="componentCount">The number of destination component planes.</param> /// <param name="componentCount">The number of destination component planes.</param>
/// <param name="precision">The JPEG sample precision.</param> /// <param name="precision">The JPEG sample precision.</param>
private static void ValidateConversionFromRgb<TOperator>( private static void ValidateConversionFromRgb<TOperator>(JpegColorConverterBase converter, int length, int componentCount, int precision)
JpegColorConverterBase converter,
int length,
int componentCount,
int precision)
where TOperator : struct, JpegColorConverterBase.IJpegColorConverterOperator where TOperator : struct, JpegColorConverterBase.IJpegColorConverterOperator
{ {
JpegColorConverterBase.ComponentValues expected = CreateRandomValues(length, componentCount, precision); JpegColorConverterBase.ComponentValues expected = CreateRandomValues(length, componentCount, precision);
@ -345,17 +316,7 @@ public class JpegColorConverterTests
for (int i = 0; i < length; i++) for (int i = 0; i < length; i++)
{ {
TOperator.ConvertFromRgb( TOperator.ConvertFromRgb(r[i], g[i], b[i], maximumValue, halfValue, scale, out expected.Component0[i], out float c1, out float c2, out float c3);
r[i],
g[i],
b[i],
maximumValue,
halfValue,
scale,
out expected.Component0[i],
out float c1,
out float c2,
out float c3);
if (componentCount >= 2) if (componentCount >= 2)
{ {
@ -399,10 +360,7 @@ public class JpegColorConverterTests
/// <param name="componentCount">The number of independent component planes.</param> /// <param name="componentCount">The number of independent component planes.</param>
/// <param name="precision">The JPEG sample precision and deterministic random seed.</param> /// <param name="precision">The JPEG sample precision and deterministic random seed.</param>
/// <returns>The generated component planes.</returns> /// <returns>The generated component planes.</returns>
private static JpegColorConverterBase.ComponentValues CreateRandomValues( private static JpegColorConverterBase.ComponentValues CreateRandomValues(int length, int componentCount, int precision)
int length,
int componentCount,
int precision)
{ {
Random random = new(precision); Random random = new(precision);
float maximumValue = MathF.Pow(2, precision) - 1; float maximumValue = MathF.Pow(2, precision) - 1;
@ -449,11 +407,7 @@ public class JpegColorConverterTests
/// <param name="source">The unmodified source component planes.</param> /// <param name="source">The unmodified source component planes.</param>
/// <param name="actual">The converted RGB planes.</param> /// <param name="actual">The converted RGB planes.</param>
/// <param name="index">The sample index.</param> /// <param name="index">The sample index.</param>
private static void AssertColorModelDefinition( private static void AssertColorModelDefinition(JpegColorSpace colorSpace, in JpegColorConverterBase.ComponentValues source, in JpegColorConverterBase.ComponentValues actual, int index)
JpegColorSpace colorSpace,
in JpegColorConverterBase.ComponentValues source,
in JpegColorConverterBase.ComponentValues actual,
int index)
{ {
float c0 = source.Component0[index]; float c0 = source.Component0[index];
float c1 = source.Component1[index]; float c1 = source.Component1[index];
@ -468,18 +422,12 @@ public class JpegColorConverterTests
expected = new Rgb(luminance, luminance, luminance); expected = new Rgb(luminance, luminance, luminance);
break; break;
case JpegColorSpace.RGB: case JpegColorSpace.RGB:
expected = new Rgb( expected = new Rgb(c0 / MaxColorChannelValue, c1 / MaxColorChannelValue, c2 / MaxColorChannelValue);
c0 / MaxColorChannelValue,
c1 / MaxColorChannelValue,
c2 / MaxColorChannelValue);
break; break;
case JpegColorSpace.Cmyk: case JpegColorSpace.Cmyk:
c3 = source.Component3[index] / MaxColorChannelValue; c3 = source.Component3[index] / MaxColorChannelValue;
expected = new Rgb( expected = new Rgb(c0 * c3 / MaxColorChannelValue, c1 * c3 / MaxColorChannelValue, c2 * c3 / MaxColorChannelValue);
c0 * c3 / MaxColorChannelValue,
c1 * c3 / MaxColorChannelValue,
c2 * c3 / MaxColorChannelValue);
break; break;
case JpegColorSpace.YCbCr: case JpegColorSpace.YCbCr:
@ -487,10 +435,7 @@ public class JpegColorConverterTests
c2 -= 128F; c2 -= 128F;
// JPEG applies the BT.601 matrix in the integer sample domain and rounds before normalization. // JPEG applies the BT.601 matrix in the integer sample domain and rounds before normalization.
expected = new Rgb( expected = new Rgb(MathF.Round(c0 + (1.402F * c2), MidpointRounding.AwayFromZero) / MaxColorChannelValue, MathF.Round(c0 - (0.344136F * c1) - (0.714136F * c2), MidpointRounding.AwayFromZero) / MaxColorChannelValue, MathF.Round(c0 + (1.772F * c1), MidpointRounding.AwayFromZero) / MaxColorChannelValue);
MathF.Round(c0 + (1.402F * c2), MidpointRounding.AwayFromZero) / MaxColorChannelValue,
MathF.Round(c0 - (0.344136F * c1) - (0.714136F * c2), MidpointRounding.AwayFromZero) / MaxColorChannelValue,
MathF.Round(c0 + (1.772F * c1), MidpointRounding.AwayFromZero) / MaxColorChannelValue);
break; break;
case JpegColorSpace.Ycck: case JpegColorSpace.Ycck:
@ -499,10 +444,7 @@ public class JpegColorConverterTests
c3 = source.Component3[index] / MaxColorChannelValue; c3 = source.Component3[index] / MaxColorChannelValue;
// Adobe YccK reconstructs inverted RGB first, then applies the normalized black component. // Adobe YccK reconstructs inverted RGB first, then applies the normalized black component.
expected = new Rgb( expected = new Rgb((MaxColorChannelValue - MathF.Round(c0 + (1.402F * c2), MidpointRounding.AwayFromZero)) * c3 / MaxColorChannelValue, (MaxColorChannelValue - MathF.Round(c0 - (0.344136F * c1) - (0.714136F * c2), MidpointRounding.AwayFromZero)) * c3 / MaxColorChannelValue, (MaxColorChannelValue - MathF.Round(c0 + (1.772F * c1), MidpointRounding.AwayFromZero)) * c3 / MaxColorChannelValue);
(MaxColorChannelValue - MathF.Round(c0 + (1.402F * c2), MidpointRounding.AwayFromZero)) * c3 / MaxColorChannelValue,
(MaxColorChannelValue - MathF.Round(c0 - (0.344136F * c1) - (0.714136F * c2), MidpointRounding.AwayFromZero)) * c3 / MaxColorChannelValue,
(MaxColorChannelValue - MathF.Round(c0 + (1.772F * c1), MidpointRounding.AwayFromZero)) * c3 / MaxColorChannelValue);
break; break;
default: default:
@ -513,12 +455,9 @@ public class JpegColorConverterTests
// Color-space comparison intentionally clamps both sides because JPEG reconstruction can overshoot // Color-space comparison intentionally clamps both sides because JPEG reconstruction can overshoot
// the normalized RGB gamut and saturation belongs to the eventual pixel conversion. // the normalized RGB gamut and saturation belongs to the eventual pixel conversion.
Rgb clampedExpected = Rgb.Clamp(expected); Rgb clampedExpected = Rgb.Clamp(expected);
Rgb clampedActual = Rgb.Clamp( Rgb clampedActual = Rgb.Clamp(new Rgb(actual.Component0[index], actual.Component1[index], actual.Component2[index]));
new Rgb(actual.Component0[index], actual.Component1[index], actual.Component2[index]));
Assert.True( Assert.True(ColorSpaceComparer.Equals(clampedExpected, clampedActual), $"Colors {clampedExpected} and {clampedActual} are not equal at index {index}.");
ColorSpaceComparer.Equals(clampedExpected, clampedActual),
$"Colors {clampedExpected} and {clampedActual} are not equal at index {index}.");
} }
/// <summary> /// <summary>

168
tests/ImageSharp.Tests/Formats/Jpg/JpegColorPackingTests.cs

@ -0,0 +1,168 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Numerics;
using SixLabors.ImageSharp.Formats.Jpeg.Components;
using SixLabors.ImageSharp.Tests.TestUtilities;
namespace SixLabors.ImageSharp.Tests.Formats.Jpg;
/// <summary>
/// Tests the planar and packed buffer transformations used around JPEG color-profile conversion.
/// </summary>
[Trait("Format", "Jpg")]
public class JpegColorPackingTests
{
private static readonly int[] Lengths = [0, 1, 2, 3, 4, 5, 7, 8, 15, 16, 17, 31, 32, 33, 129];
/// <summary>
/// Verifies every packing operation against its scalar definition with and without hardware intrinsics.
/// </summary>
[Fact]
public void PackingOperationsMatchScalarDefinitions()
=> FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidatePackingOperations, HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic);
/// <summary>
/// Exercises every SIMD transition and scalar remainder for the packing operations.
/// </summary>
private static void ValidatePackingOperations()
{
foreach (int length in Lengths)
{
ValidatePackedNormalizeInterleave3(length);
ValidateUnpackDeinterleave3(length);
ValidatePackedNormalizeInterleave4(length);
ValidatePackedInvertNormalizeInterleave4(length);
}
}
/// <summary>
/// Compares normalized three-plane interleaving with the original scalar loop.
/// </summary>
/// <param name="length">The number of samples in each component plane.</param>
private static void ValidatePackedNormalizeInterleave3(int length)
{
const float scale = 1F / 255F;
float[] x = CreateSamples(length, 1);
float[] y = CreateSamples(length, 2);
float[] z = CreateSamples(length, 3);
float[] expected = new float[length * 3];
float[] actual = new float[length * 3];
for (int i = 0; i < length; i++)
{
int packedOffset = i * 3;
expected[packedOffset] = x[i] * scale;
expected[packedOffset + 1] = y[i] * scale;
expected[packedOffset + 2] = z[i] * scale;
}
JpegColorConverterBase.PackedNormalizeInterleave3(x, y, z, actual, scale);
Assert.Equal(expected, actual);
}
/// <summary>
/// Compares packed three-channel deinterleaving with the original scalar loop.
/// </summary>
/// <param name="length">The number of packed values.</param>
private static void ValidateUnpackDeinterleave3(int length)
{
Vector3[] packed = new Vector3[length];
float[] expectedX = CreateSamples(length, 1);
float[] expectedY = CreateSamples(length, 2);
float[] expectedZ = CreateSamples(length, 3);
float[] actualX = new float[length];
float[] actualY = new float[length];
float[] actualZ = new float[length];
for (int i = 0; i < length; i++)
{
packed[i] = new Vector3(expectedX[i], expectedY[i], expectedZ[i]);
}
JpegColorConverterBase.UnpackDeinterleave3(packed, actualX, actualY, actualZ);
Assert.Equal(expectedX, actualX);
Assert.Equal(expectedY, actualY);
Assert.Equal(expectedZ, actualZ);
}
/// <summary>
/// Compares normalized four-plane interleaving with the original scalar loop.
/// </summary>
/// <param name="length">The number of samples in each component plane.</param>
private static void ValidatePackedNormalizeInterleave4(int length)
{
const float maximumValue = 255F;
const float scale = 1F / maximumValue;
float[] x = CreateSamples(length, 1);
float[] y = CreateSamples(length, 2);
float[] z = CreateSamples(length, 3);
float[] w = CreateSamples(length, 4);
float[] expected = new float[length * 4];
float[] actual = new float[length * 4];
for (int i = 0; i < length; i++)
{
int packedOffset = i * 4;
expected[packedOffset] = x[i] * scale;
expected[packedOffset + 1] = y[i] * scale;
expected[packedOffset + 2] = z[i] * scale;
expected[packedOffset + 3] = w[i] * scale;
}
JpegColorConverterBase.PackedNormalizeInterleave4(x, y, z, w, actual, maximumValue);
Assert.Equal(expected, actual);
}
/// <summary>
/// Compares inverted normalized four-plane interleaving with the original scalar loop.
/// </summary>
/// <param name="length">The number of samples in each component plane.</param>
private static void ValidatePackedInvertNormalizeInterleave4(int length)
{
const float maximumValue = 255F;
const float scale = 1F / maximumValue;
float[] x = CreateSamples(length, 1);
float[] y = CreateSamples(length, 2);
float[] z = CreateSamples(length, 3);
float[] w = CreateSamples(length, 4);
float[] expected = new float[length * 4];
float[] actual = new float[length * 4];
for (int i = 0; i < length; i++)
{
int packedOffset = i * 4;
expected[packedOffset] = (maximumValue - x[i]) * scale;
expected[packedOffset + 1] = (maximumValue - y[i]) * scale;
expected[packedOffset + 2] = (maximumValue - z[i]) * scale;
expected[packedOffset + 3] = (maximumValue - w[i]) * scale;
}
JpegColorConverterBase.PackedInvertNormalizeInterleave4(x, y, z, w, actual, maximumValue);
Assert.Equal(expected, actual);
}
/// <summary>
/// Creates deterministic, non-integral sample values that expose lane-order and arithmetic mistakes.
/// </summary>
/// <param name="length">The number of samples to create.</param>
/// <param name="component">The one-based component number used to distinguish each plane.</param>
/// <returns>The generated sample values.</returns>
private static float[] CreateSamples(int length, int component)
{
float[] samples = new float[length];
for (int i = 0; i < samples.Length; i++)
{
// The relatively prime multipliers produce a distinct sequence for each plane
// while keeping every value inside the eight-bit JPEG sample domain.
samples[i] = (((i * 37) + (component * 53)) % 251) + (component * 0.125F);
}
return samples;
}
}

79
tests/ImageSharp.Tests/Formats/Png/PngEncoderFilterTests.cs

@ -63,9 +63,7 @@ public class PngEncoderFilterTests : MeasureFixture
data.TestFilter(); data.TestFilter();
} }
FeatureTestRunner.RunWithHwIntrinsicsFeature( FeatureTestRunner.RunWithHwIntrinsicsFeature(RunTest, HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX2);
RunTest,
HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX2);
} }
[Fact] [Fact]
@ -213,12 +211,7 @@ public class PngEncoderFilterTests : MeasureFixture
/// </summary> /// </summary>
[Fact] [Fact]
public void EncodeMatchesReferencesAcrossRegisterBoundaries() public void EncodeMatchesReferencesAcrossRegisterBoundaries()
=> FeatureTestRunner.RunWithHwIntrinsicsFeature( => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertEncodersMatchReferences, HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX2 | HwIntrinsics.DisableHWIntrinsic);
AssertEncodersMatchReferences,
HwIntrinsics.AllowAll
| HwIntrinsics.DisableAVX512F
| HwIntrinsics.DisableAVX2
| HwIntrinsics.DisableHWIntrinsic);
/// <summary> /// <summary>
/// Compares every filter with its independent scalar reference across SIMD boundaries and pixel strides. /// Compares every filter with its independent scalar reference across SIMD boundaries and pixel strides.
@ -243,29 +236,13 @@ public class PngEncoderFilterTests : MeasureFixture
random.NextBytes(scanline); random.NextBytes(scanline);
random.NextBytes(previousScanline); random.NextBytes(previousScanline);
AssertFilterMatchesReference( AssertFilterMatchesReference(PngFilterMethod.Sub, scanline, previousScanline, bytesPerPixel);
PngFilterMethod.Sub,
scanline, AssertFilterMatchesReference(PngFilterMethod.Up, scanline, previousScanline, bytesPerPixel);
previousScanline,
bytesPerPixel); AssertFilterMatchesReference(PngFilterMethod.Average, scanline, previousScanline, bytesPerPixel);
AssertFilterMatchesReference( AssertFilterMatchesReference(PngFilterMethod.Paeth, scanline, previousScanline, bytesPerPixel);
PngFilterMethod.Up,
scanline,
previousScanline,
bytesPerPixel);
AssertFilterMatchesReference(
PngFilterMethod.Average,
scanline,
previousScanline,
bytesPerPixel);
AssertFilterMatchesReference(
PngFilterMethod.Paeth,
scanline,
previousScanline,
bytesPerPixel);
} }
} }
} }
@ -277,11 +254,7 @@ public class PngEncoderFilterTests : MeasureFixture
/// <param name="scanline">The current scanline.</param> /// <param name="scanline">The current scanline.</param>
/// <param name="previousScanline">The preceding scanline.</param> /// <param name="previousScanline">The preceding scanline.</param>
/// <param name="bytesPerPixel">The component distance between adjacent pixels.</param> /// <param name="bytesPerPixel">The component distance between adjacent pixels.</param>
private static void AssertFilterMatchesReference( private static void AssertFilterMatchesReference(PngFilterMethod filter, byte[] scanline, byte[] previousScanline, int bytesPerPixel)
PngFilterMethod filter,
byte[] scanline,
byte[] previousScanline,
int bytesPerPixel)
{ {
byte[] expected = new byte[scanline.Length + 1]; byte[] expected = new byte[scanline.Length + 1];
byte[] actual = new byte[scanline.Length + 1]; byte[] actual = new byte[scanline.Length + 1];
@ -301,36 +274,16 @@ public class PngEncoderFilterTests : MeasureFixture
break; break;
case PngFilterMethod.Average: case PngFilterMethod.Average:
ReferenceImplementations.EncodeAverageFilter( ReferenceImplementations.EncodeAverageFilter(scanline, previousScanline, expected, bytesPerPixel, out expectedSum);
scanline,
previousScanline, AverageFilter.Encode(scanline, previousScanline, actual, (uint)bytesPerPixel, out actualSum);
expected,
bytesPerPixel,
out expectedSum);
AverageFilter.Encode(
scanline,
previousScanline,
actual,
(uint)bytesPerPixel,
out actualSum);
break; break;
case PngFilterMethod.Paeth: case PngFilterMethod.Paeth:
ReferenceImplementations.EncodePaethFilter( ReferenceImplementations.EncodePaethFilter(scanline, previousScanline, expected, bytesPerPixel, out expectedSum);
scanline,
previousScanline, PaethFilter.Encode(scanline, previousScanline, actual, bytesPerPixel, out actualSum);
expected,
bytesPerPixel,
out expectedSum);
PaethFilter.Encode(
scanline,
previousScanline,
actual,
bytesPerPixel,
out actualSum);
break; break;

16
tests/ImageSharp.Tests/PixelFormats/PixelBlenderTests.cs

@ -866,12 +866,7 @@ public class PixelBlenderTests
/// <param name="amount">The source opacity.</param> /// <param name="amount">The source opacity.</param>
/// <param name="coverage">The pixel coverage.</param> /// <param name="coverage">The pixel coverage.</param>
/// <returns>The blended pixel.</returns> /// <returns>The blended pixel.</returns>
private static TPixel BlendWithCoverageScalar<TPixel>( private static TPixel BlendWithCoverageScalar<TPixel>(PixelBlender<TPixel> blender, TPixel background, TPixel source, float amount, float coverage)
PixelBlender<TPixel> blender,
TPixel background,
TPixel source,
float amount,
float coverage)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
Span<TPixel> destination = stackalloc TPixel[1]; Span<TPixel> destination = stackalloc TPixel[1];
@ -880,14 +875,7 @@ public class PixelBlenderTests
Span<float> coverageSpan = stackalloc float[1] { coverage }; Span<float> coverageSpan = stackalloc float[1] { coverage };
Span<Vector4> buffer = stackalloc Vector4[3]; Span<Vector4> buffer = stackalloc Vector4[3];
blender.BlendWithCoverage<TPixel>( blender.BlendWithCoverage<TPixel>(Configuration.Default, destination, backgroundSpan, sourceSpan, amount, coverageSpan, buffer);
Configuration.Default,
destination,
backgroundSpan,
sourceSpan,
amount,
coverageSpan,
buffer);
return destination[0]; return destination[0];
} }

32
tests/ImageSharp.Tests/PixelFormats/Vector4ConvertersTests.cs

@ -20,24 +20,14 @@ public class Vector4ConvertersTests
/// </summary> /// </summary>
[Fact] [Fact]
public void MultiplyThenAddMatchesComponentArithmeticAcrossHardwareWidths() public void MultiplyThenAddMatchesComponentArithmeticAcrossHardwareWidths()
=> FeatureTestRunner.RunWithHwIntrinsicsFeature( => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertMultiplyThenAddMatchesComponentArithmetic, HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic);
AssertMultiplyThenAddMatchesComponentArithmetic,
HwIntrinsics.AllowAll
| HwIntrinsics.DisableAVX512F
| HwIntrinsics.DisableAVX
| HwIntrinsics.DisableHWIntrinsic);
/// <summary> /// <summary>
/// Verifies add-then-divide behavior for every SIMD boundary and the software fallback. /// Verifies add-then-divide behavior for every SIMD boundary and the software fallback.
/// </summary> /// </summary>
[Fact] [Fact]
public void AddThenDivideMatchesComponentArithmeticAcrossHardwareWidths() public void AddThenDivideMatchesComponentArithmeticAcrossHardwareWidths()
=> FeatureTestRunner.RunWithHwIntrinsicsFeature( => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertAddThenDivideMatchesComponentArithmetic, HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic);
AssertAddThenDivideMatchesComponentArithmetic,
HwIntrinsics.AllowAll
| HwIntrinsics.DisableAVX512F
| HwIntrinsics.DisableAVX
| HwIntrinsics.DisableHWIntrinsic);
/// <summary> /// <summary>
/// Compares the multiply-then-add traversal with independently evaluated component expressions. /// Compares the multiply-then-add traversal with independently evaluated component expressions.
@ -56,11 +46,7 @@ public class Vector4ConvertersTests
{ {
Vector4 value = actual[i]; Vector4 value = actual[i];
expected[i] = new Vector4( expected[i] = new Vector4((value.X * multiplier.X) + offset.X, (value.Y * multiplier.Y) + offset.Y, (value.Z * multiplier.Z) + offset.Z, (value.W * multiplier.W) + offset.W);
(value.X * multiplier.X) + offset.X,
(value.Y * multiplier.Y) + offset.Y,
(value.Z * multiplier.Z) + offset.Z,
(value.W * multiplier.W) + offset.W);
} }
Vector4Converters.MultiplyThenAdd(actual, multiplier, offset); Vector4Converters.MultiplyThenAdd(actual, multiplier, offset);
@ -86,11 +72,7 @@ public class Vector4ConvertersTests
{ {
Vector4 value = actual[i]; Vector4 value = actual[i];
expected[i] = new Vector4( expected[i] = new Vector4((value.X + offset.X) / divisor.X, (value.Y + offset.Y) / divisor.Y, (value.Z + offset.Z) / divisor.Z, (value.W + offset.W) / divisor.W);
(value.X + offset.X) / divisor.X,
(value.Y + offset.Y) / divisor.Y,
(value.Z + offset.Z) / divisor.Z,
(value.W + offset.W) / divisor.W);
} }
Vector4Converters.AddThenDivide(actual, offset, divisor); Vector4Converters.AddThenDivide(actual, offset, divisor);
@ -110,11 +92,7 @@ public class Vector4ConvertersTests
for (int i = 0; i < result.Length; i++) for (int i = 0; i < result.Length; i++)
{ {
result[i] = new Vector4( result[i] = new Vector4((i * 17F) - 31F, (i * -23F) + 37F, (i * .25F) - 41F, (i * 3F) + 43F);
(i * 17F) - 31F,
(i * -23F) + 37F,
(i * .25F) - 41F,
(i * 3F) + 43F);
} }
return result; return result;

Loading…
Cancel
Save