diff --git a/src/ImageSharp/ColorSpaces/Companding/SRgbCompanding.cs b/src/ImageSharp/ColorSpaces/Companding/SRgbCompanding.cs index 9a8b5f0a8..865c28f3d 100644 --- a/src/ImageSharp/ColorSpaces/Companding/SRgbCompanding.cs +++ b/src/ImageSharp/ColorSpaces/Companding/SRgbCompanding.cs @@ -5,6 +5,10 @@ using System; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if SUPPORTS_RUNTIME_INTRINSICS +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +#endif namespace SixLabors.ImageSharp.ColorSpaces.Companding { @@ -18,21 +22,78 @@ namespace SixLabors.ImageSharp.ColorSpaces.Companding /// public static class SRgbCompanding { + private const int Length = Scale + 2; + private const int Scale = (1 << 14) - 1; + + private static readonly Lazy LazyCompressTable = new Lazy(() => + { + var result = new float[Length]; + + for (int i = 0; i < result.Length; i++) + { + double d = (double)i / Scale; + if (d <= (0.04045 / 12.92)) + { + d *= 12.92; + } + else + { + d = (1.055 * Math.Pow(d, 1.0 / 2.4)) - 0.055; + } + + result[i] = (float)d; + } + + return result; + }); + + private static readonly Lazy LazyExpandTable = new Lazy(() => + { + var result = new float[Length]; + + for (int i = 0; i < result.Length; i++) + { + double d = (double)i / Scale; + if (d <= 0.04045) + { + d /= 12.92; + } + else + { + d = Math.Pow((d + 0.055) / 1.055, 2.4); + } + + result[i] = (float)d; + } + + return result; + }); + + private static readonly float[] ExpandTable = LazyExpandTable.Value; + private static readonly float[] CompressTable = LazyCompressTable.Value; + /// /// Expands the companded vectors to their linear equivalents with respect to the energy. /// /// The span of vectors. - [MethodImpl(InliningOptions.ShortMethod)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Expand(Span vectors) { - ref Vector4 vectorsStart = ref MemoryMarshal.GetReference(vectors); - ref Vector4 vectorsEnd = ref Unsafe.Add(ref vectorsStart, vectors.Length); - - while (Unsafe.IsAddressLessThan(ref vectorsStart, ref vectorsEnd)) +#if SUPPORTS_RUNTIME_INTRINSICS + if (Avx2.IsSupported && vectors.Length >= 2) { - Expand(ref vectorsStart); + CompandAvx2(vectors, ExpandTable); - vectorsStart = ref Unsafe.Add(ref vectorsStart, 1); + if (Numerics.Modulo2(vectors.Length) != 0) + { + // Vector4 fits neatly in pairs. Any overlap has to be equal to 1. + Expand(ref MemoryMarshal.GetReference(vectors.Slice(vectors.Length - 1))); + } + } + else +#endif + { + CompandScalar(vectors, ExpandTable); } } @@ -40,17 +101,24 @@ namespace SixLabors.ImageSharp.ColorSpaces.Companding /// Compresses the uncompanded vectors to their nonlinear equivalents with respect to the energy. /// /// The span of vectors. - [MethodImpl(InliningOptions.ShortMethod)] - public static void Compress(Span vectors) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe void Compress(Span vectors) { - ref Vector4 vectorsStart = ref MemoryMarshal.GetReference(vectors); - ref Vector4 vectorsEnd = ref Unsafe.Add(ref vectorsStart, vectors.Length); - - while (Unsafe.IsAddressLessThan(ref vectorsStart, ref vectorsEnd)) +#if SUPPORTS_RUNTIME_INTRINSICS + if (Avx2.IsSupported && vectors.Length >= 2) { - Compress(ref vectorsStart); + CompandAvx2(vectors, CompressTable); - vectorsStart = ref Unsafe.Add(ref vectorsStart, 1); + if (Numerics.Modulo2(vectors.Length) != 0) + { + // Vector4 fits neatly in pairs. Any overlap has to be equal to 1. + Compress(ref MemoryMarshal.GetReference(vectors.Slice(vectors.Length - 1))); + } + } + else +#endif + { + CompandScalar(vectors, CompressTable); } } @@ -58,24 +126,26 @@ namespace SixLabors.ImageSharp.ColorSpaces.Companding /// Expands a companded vector to its linear equivalent with respect to the energy. /// /// The vector. - [MethodImpl(InliningOptions.ShortMethod)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Expand(ref Vector4 vector) { vector.X = Expand(vector.X); vector.Y = Expand(vector.Y); vector.Z = Expand(vector.Z); + vector.W = Expand(vector.W); } /// /// Compresses an uncompanded vector (linear) to its nonlinear equivalent. /// /// The vector. - [MethodImpl(InliningOptions.ShortMethod)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Compress(ref Vector4 vector) { vector.X = Compress(vector.X); vector.Y = Compress(vector.Y); vector.Z = Compress(vector.Z); + vector.W = Compress(vector.W); } /// @@ -83,15 +153,84 @@ namespace SixLabors.ImageSharp.ColorSpaces.Companding /// /// The channel value. /// The representing the linear channel value. - [MethodImpl(InliningOptions.ShortMethod)] - public static float Expand(float channel) => channel <= 0.04045F ? channel / 12.92F : MathF.Pow((channel + 0.055F) / 1.055F, 2.4F); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Expand(float channel) + => channel <= 0.04045F ? channel / 12.92F : MathF.Pow((channel + 0.055F) / 1.055F, 2.4F); /// /// Compresses an uncompanded channel (linear) to its nonlinear equivalent. /// /// The channel value. /// The representing the nonlinear channel value. - [MethodImpl(InliningOptions.ShortMethod)] - public static float Compress(float channel) => channel <= 0.0031308F ? 12.92F * channel : (1.055F * MathF.Pow(channel, 0.416666666666667F)) - 0.055F; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Compress(float channel) + => channel <= 0.0031308F ? 12.92F * channel : (1.055F * MathF.Pow(channel, 0.416666666666667F)) - 0.055F; + +#if SUPPORTS_RUNTIME_INTRINSICS + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void CompandAvx2(Span vectors, float[] table) + { + fixed (float* tablePointer = &table[0]) + { + var scale = Vector256.Create((float)Scale); + Vector256 zero = Vector256.Zero; + var offset = Vector256.Create(1); + + // Divide by 2 as 4 elements per Vector4 and 8 per Vector256 + ref Vector256 vectorsBase = ref Unsafe.As>(ref MemoryMarshal.GetReference(vectors)); + ref Vector256 vectorsLast = ref Unsafe.Add(ref vectorsBase, (IntPtr)((uint)vectors.Length / 2u)); + + while (Unsafe.IsAddressLessThan(ref vectorsBase, ref vectorsLast)) + { + Vector256 multiplied = Avx.Multiply(scale, vectorsBase); + multiplied = Avx.Min(Avx.Max(zero, multiplied), scale); + + Vector256 truncated = Avx.ConvertToVector256Int32WithTruncation(multiplied); + Vector256 truncatedF = Avx.ConvertToVector256Single(truncated); + + Vector256 low = Avx2.GatherVector256(tablePointer, truncated, sizeof(float)); + Vector256 high = Avx2.GatherVector256(tablePointer, Avx2.Add(truncated, offset), sizeof(float)); + + vectorsBase = Numerics.Lerp(low, high, Avx.Subtract(multiplied, truncatedF)); + vectorsBase = ref Unsafe.Add(ref vectorsBase, 1); + } + } + } +#endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void CompandScalar(Span vectors, float[] table) + { + fixed (float* tablePointer = &table[0]) + { + Vector4 zero = Vector4.Zero; + var scale = new Vector4(Scale); + ref Vector4 vectorsBase = ref MemoryMarshal.GetReference(vectors); + ref Vector4 vectorsLast = ref Unsafe.Add(ref vectorsBase, vectors.Length); + + while (Unsafe.IsAddressLessThan(ref vectorsBase, ref vectorsLast)) + { + Vector4 multiplied = Numerics.Clamp(vectorsBase * Scale, zero, scale); + + float f0 = multiplied.X; + float f1 = multiplied.Y; + float f2 = multiplied.Z; + float f3 = multiplied.W; + + uint i0 = (uint)f0; + uint i1 = (uint)f1; + uint i2 = (uint)f2; + uint i3 = (uint)f3; + + vectorsBase.X = Numerics.Lerp(tablePointer[i0], tablePointer[i0 + 1], f0 - (int)i0); + vectorsBase.Y = Numerics.Lerp(tablePointer[i1], tablePointer[i1 + 1], f1 - (int)i1); + vectorsBase.Z = Numerics.Lerp(tablePointer[i2], tablePointer[i2 + 1], f2 - (int)i2); + vectorsBase.W = Numerics.Lerp(tablePointer[i3], tablePointer[i3 + 1], f3 - (int)i3); + + vectorsBase = ref Unsafe.Add(ref vectorsBase, 1); + } + } + } } } diff --git a/src/ImageSharp/Common/Helpers/Numerics.cs b/src/ImageSharp/Common/Helpers/Numerics.cs index b9ccfafe0..0ff8b3082 100644 --- a/src/ImageSharp/Common/Helpers/Numerics.cs +++ b/src/ImageSharp/Common/Helpers/Numerics.cs @@ -710,5 +710,43 @@ namespace SixLabors.ImageSharp } } } + +#if SUPPORTS_RUNTIME_INTRINSICS + + /// + /// Performs a linear interpolation between two values based on the given weighting. + /// + /// The first value. + /// The second value. + /// Values between 0 and 1 that indicates the weight of . + /// The . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Lerp( + in Vector256 value1, + in Vector256 value2, + in Vector256 amount) + { + Vector256 diff = Avx.Subtract(value2, value1); + if (Fma.IsSupported) + { + return Fma.MultiplyAdd(diff, amount, value1); + } + else + { + return Avx.Add(Avx.Multiply(diff, amount), value1); + } + } +#endif + + /// + /// Performs a linear interpolation between two values based on the given weighting. + /// + /// The first value. + /// The second value. + /// A value between 0 and 1 that indicates the weight of . + /// The . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Lerp(float value1, float value2, float amount) + => ((value2 - value1) * amount) + value1; } }