diff --git a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs index 760296c9d3..4492b297ce 100644 --- a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs +++ b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs @@ -1300,4 +1300,68 @@ internal static class Vector128_ // Narrow back to signed bytes return Vector128.Narrow(diffLo, diffHi); } + + /// + /// Create mask from the most significant bit of each 8-bit element in , and store the result. + /// + /// + /// The vector containing packed 8-bit integers from which to create the mask. + /// + /// + /// A 16-bit integer mask where each bit corresponds to the most significant bit of each 8-bit element + /// in . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int MoveMask(Vector128 value) + { + if (Sse2.IsSupported) + { + return Sse2.MoveMask(value); + } + + if (AdvSimd.IsSupported) + { + // https://stackoverflow.com/questions/11870910/sse-mm-movemask-epi8-equivalent-method-for-arm-neon + Vector128 powers = Vector128.Create(1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128); + Vector128 masked = value & powers; + + Vector128 sum8 = AdvSimd.AddPairwiseWidening(masked); + Vector128 sum16 = AdvSimd.AddPairwiseWidening(sum8); + Vector128 sum32 = AdvSimd.AddPairwiseWidening(sum16); + + // Extract lower 8 bits of each 64-bit lane + byte lo = sum32.AsByte().GetElement(0); + byte hi = sum32.AsByte().GetElement(8); + + return (hi << 8) | lo; + } + + { + // Step 1: isolate MSBs + Vector128 msbMask = Vector128.Create((byte)0x80); + Vector128 masked = value & msbMask; + + // Step 2: shift each byte so MSB lands in bit position [0..15] + // i.e. convert: 0x80 → 1 << i + Vector128 bitShifts = Vector128.Create((ushort)1, 2, 4, 8, 16, 32, 64, 128); + Vector128 bitShiftsHigh = Vector128.Create(256, 512, 1024, 2048, 4096, 8192, 16384, 32768); + + // Step 3: widen to ushort + (Vector128 lo, Vector128 hi) = Vector128.Widen(masked); + + // Step 4: compare > 0 to get 0xFFFF where MSB was set + lo = Vector128.ConditionalSelect(Vector128.Equals(lo, Vector128.Zero), Vector128.Zero, bitShifts); + hi = Vector128.ConditionalSelect(Vector128.Equals(hi, Vector128.Zero), Vector128.Zero, bitShiftsHigh); + + // Step 5: bitwise OR the two halves + Vector128 maskVector = lo | hi; + + // Step 6: horizontal OR reduction via shuffles + maskVector |= Vector128.Shuffle(maskVector, Vector128.Create((ushort)4, 5, 6, 7, 0, 1, 2, 3)); + maskVector |= Vector128.Shuffle(maskVector, Vector128.Create((ushort)2, 3, 0, 1, 6, 7, 4, 5)); + maskVector |= Vector128.Shuffle(maskVector, Vector128.Create((ushort)1, 0, 3, 2, 5, 4, 7, 6)); + + return maskVector.ToScalar(); + } + } } diff --git a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs index 71dfadc399..e61dcb6bf9 100644 --- a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs +++ b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs @@ -231,6 +231,27 @@ internal static class Vector256_ return Vector256.Narrow(lefClamped, rightClamped); } + /// + /// Packs signed 16-bit integers to signed 8-bit integers and saturates. + /// + /// The left hand source vector. + /// The right hand source vector. + /// The . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 PackSignedSaturate(Vector256 left, Vector256 right) + { + if (Avx2.IsSupported) + { + return Avx2.PackSignedSaturate(left, right); + } + + Vector256 min = Vector256.Create((short)sbyte.MinValue); + Vector256 max = Vector256.Create((short)sbyte.MaxValue); + Vector256 lefClamped = Clamp(left, min, max); + Vector256 rightClamped = Clamp(right, min, max); + return Vector256.Narrow(lefClamped, rightClamped); + } + /// /// Restricts a vector between a minimum and a maximum value. /// @@ -466,6 +487,28 @@ internal static class Vector256_ Vector128_.SubtractSaturate(left.GetUpper(), right.GetUpper())); } + /// + /// Create mask from the most significant bit of each 8-bit element in , and store the result. + /// + /// + /// The vector containing packed 8-bit integers from which to create the mask. + /// + /// + /// A 16-bit integer mask where each bit corresponds to the most significant bit of each 8-bit element + /// in . + /// + public static int MoveMask(Vector256 value) + { + if (Avx2.IsSupported) + { + return Avx2.MoveMask(value); + } + + int loMask = Vector128_.MoveMask(value.GetLower()); + int hiMask = Vector128_.MoveMask(value.GetUpper()); + return loMask | (hiMask << 16); + } + [DoesNotReturn] private static void ThrowUnreachableException() => throw new UnreachableException(); } diff --git a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs index 43dab1ffc4..c7ce12fc7b 100644 --- a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs +++ b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs @@ -6,7 +6,6 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Formats.Webp.BitReader; @@ -314,7 +313,7 @@ internal class AlphaDecoder : IDisposable private static void HorizontalUnfilter(Span prev, Span input, Span dst, int width) { - if ((Sse2.IsSupported || AdvSimd.IsSupported) && width >= 9) + if (Vector128.IsHardwareAccelerated && width >= 9) { dst[0] = (byte)(input[0] + (prev.IsEmpty ? 0 : prev[0])); nuint i; @@ -362,7 +361,7 @@ internal class AlphaDecoder : IDisposable { HorizontalUnfilter(null, input, dst, width); } - else if (Avx2.IsSupported) + else if (Vector256.IsHardwareAccelerated) { ref byte inputRef = ref MemoryMarshal.GetReference(input); ref byte prevRef = ref MemoryMarshal.GetReference(prev); @@ -374,7 +373,7 @@ internal class AlphaDecoder : IDisposable { Vector256 a0 = Unsafe.As>(ref Unsafe.Add(ref inputRef, i)); Vector256 b0 = Unsafe.As>(ref Unsafe.Add(ref prevRef, i)); - Vector256 c0 = Avx2.Add(a0.AsByte(), b0.AsByte()); + Vector256 c0 = a0.AsByte() + b0.AsByte(); ref byte outputRef = ref Unsafe.Add(ref dstRef, i); Unsafe.As>(ref outputRef) = c0; } diff --git a/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs b/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs index 1ca409f9a4..b08fe15f51 100644 --- a/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs +++ b/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs @@ -3,7 +3,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Formats.Webp; @@ -20,7 +20,7 @@ internal static class WebpCommonUtils /// Returns true if alpha has non-0xff values. public static unsafe bool CheckNonOpaque(ReadOnlySpan row) { - if (Avx2.IsSupported) + if (Vector256.IsHardwareAccelerated) { ReadOnlySpan rowBytes = MemoryMarshal.AsBytes(row); int i = 0; @@ -32,19 +32,19 @@ internal static class WebpCommonUtils for (; i + 128 <= length; i += 128) { - Vector256 a0 = Avx.LoadVector256(src + i).AsByte(); - Vector256 a1 = Avx.LoadVector256(src + i + 32).AsByte(); - Vector256 a2 = Avx.LoadVector256(src + i + 64).AsByte(); - Vector256 a3 = Avx.LoadVector256(src + i + 96).AsByte(); - Vector256 b0 = Avx2.And(a0, alphaMaskVector256).AsInt32(); - Vector256 b1 = Avx2.And(a1, alphaMaskVector256).AsInt32(); - Vector256 b2 = Avx2.And(a2, alphaMaskVector256).AsInt32(); - Vector256 b3 = Avx2.And(a3, alphaMaskVector256).AsInt32(); - Vector256 c0 = Avx2.PackSignedSaturate(b0, b1).AsInt16(); - Vector256 c1 = Avx2.PackSignedSaturate(b2, b3).AsInt16(); - Vector256 d = Avx2.PackSignedSaturate(c0, c1).AsByte(); - Vector256 bits = Avx2.CompareEqual(d, all0x80Vector256); - int mask = Avx2.MoveMask(bits); + Vector256 a0 = Vector256.Load(src + i).AsByte(); + Vector256 a1 = Vector256.Load(src + i + 32).AsByte(); + Vector256 a2 = Vector256.Load(src + i + 64).AsByte(); + Vector256 a3 = Vector256.Load(src + i + 96).AsByte(); + Vector256 b0 = (a0 & alphaMaskVector256).AsInt32(); + Vector256 b1 = (a1 & alphaMaskVector256).AsInt32(); + Vector256 b2 = (a2 & alphaMaskVector256).AsInt32(); + Vector256 b3 = (a3 & alphaMaskVector256).AsInt32(); + Vector256 c0 = Vector256_.PackSignedSaturate(b0, b1).AsInt16(); + Vector256 c1 = Vector256_.PackSignedSaturate(b2, b3).AsInt16(); + Vector256 d = Vector256_.PackSignedSaturate(c0, c1).AsByte(); + Vector256 bits = Vector256.Equals(d, all0x80Vector256); + int mask = Vector256_.MoveMask(bits); if (mask != -1) { return true; @@ -53,7 +53,7 @@ internal static class WebpCommonUtils for (; i + 64 <= length; i += 64) { - if (IsNoneOpaque64Bytes(src, i)) + if (IsNoneOpaque64BytesVector128(src, i)) { return true; } @@ -61,7 +61,7 @@ internal static class WebpCommonUtils for (; i + 32 <= length; i += 32) { - if (IsNoneOpaque32Bytes(src, i)) + if (IsNonOpaque32BytesVector128(src, i)) { return true; } @@ -76,7 +76,7 @@ internal static class WebpCommonUtils } } } - else if (Sse2.IsSupported) + else if (Vector128.IsHardwareAccelerated) { ReadOnlySpan rowBytes = MemoryMarshal.AsBytes(row); int i = 0; @@ -85,7 +85,7 @@ internal static class WebpCommonUtils { for (; i + 64 <= length; i += 64) { - if (IsNoneOpaque64Bytes(src, i)) + if (IsNoneOpaque64BytesVector128(src, i)) { return true; } @@ -93,7 +93,7 @@ internal static class WebpCommonUtils for (; i + 32 <= length; i += 32) { - if (IsNoneOpaque32Bytes(src, i)) + if (IsNonOpaque32BytesVector128(src, i)) { return true; } @@ -122,38 +122,38 @@ internal static class WebpCommonUtils return false; } - private static unsafe bool IsNoneOpaque64Bytes(byte* src, int i) + private static unsafe bool IsNoneOpaque64BytesVector128(byte* src, int i) { Vector128 alphaMask = Vector128.Create(0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255); - Vector128 a0 = Sse2.LoadVector128(src + i).AsByte(); - Vector128 a1 = Sse2.LoadVector128(src + i + 16).AsByte(); - Vector128 a2 = Sse2.LoadVector128(src + i + 32).AsByte(); - Vector128 a3 = Sse2.LoadVector128(src + i + 48).AsByte(); - Vector128 b0 = Sse2.And(a0, alphaMask).AsInt32(); - Vector128 b1 = Sse2.And(a1, alphaMask).AsInt32(); - Vector128 b2 = Sse2.And(a2, alphaMask).AsInt32(); - Vector128 b3 = Sse2.And(a3, alphaMask).AsInt32(); - Vector128 c0 = Sse2.PackSignedSaturate(b0, b1).AsInt16(); - Vector128 c1 = Sse2.PackSignedSaturate(b2, b3).AsInt16(); - Vector128 d = Sse2.PackSignedSaturate(c0, c1).AsByte(); - Vector128 bits = Sse2.CompareEqual(d, Vector128.Create((byte)0x80).AsByte()); - int mask = Sse2.MoveMask(bits); + Vector128 a0 = Vector128.Load(src + i).AsByte(); + Vector128 a1 = Vector128.Load(src + i + 16).AsByte(); + Vector128 a2 = Vector128.Load(src + i + 32).AsByte(); + Vector128 a3 = Vector128.Load(src + i + 48).AsByte(); + Vector128 b0 = (a0 & alphaMask).AsInt32(); + Vector128 b1 = (a1 & alphaMask).AsInt32(); + Vector128 b2 = (a2 & alphaMask).AsInt32(); + Vector128 b3 = (a3 & alphaMask).AsInt32(); + Vector128 c0 = Vector128_.PackSignedSaturate(b0, b1).AsInt16(); + Vector128 c1 = Vector128_.PackSignedSaturate(b2, b3).AsInt16(); + Vector128 d = Vector128_.PackSignedSaturate(c0, c1).AsByte(); + Vector128 bits = Vector128.Equals(d, Vector128.Create((byte)0x80).AsByte()); + int mask = Vector128_.MoveMask(bits); return mask != 0xFFFF; } - private static unsafe bool IsNoneOpaque32Bytes(byte* src, int i) + private static unsafe bool IsNonOpaque32BytesVector128(byte* src, int i) { Vector128 alphaMask = Vector128.Create(0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255); - Vector128 a0 = Sse2.LoadVector128(src + i).AsByte(); - Vector128 a1 = Sse2.LoadVector128(src + i + 16).AsByte(); - Vector128 b0 = Sse2.And(a0, alphaMask).AsInt32(); - Vector128 b1 = Sse2.And(a1, alphaMask).AsInt32(); - Vector128 c = Sse2.PackSignedSaturate(b0, b1).AsInt16(); - Vector128 d = Sse2.PackSignedSaturate(c, c).AsByte(); - Vector128 bits = Sse2.CompareEqual(d, Vector128.Create((byte)0x80).AsByte()); - int mask = Sse2.MoveMask(bits); + Vector128 a0 = Vector128.Load(src + i).AsByte(); + Vector128 a1 = Vector128.Load(src + i + 16).AsByte(); + Vector128 b0 = (a0 & alphaMask).AsInt32(); + Vector128 b1 = (a1 & alphaMask).AsInt32(); + Vector128 c = Vector128_.PackSignedSaturate(b0, b1).AsInt16(); + Vector128 d = Vector128_.PackSignedSaturate(c, c).AsByte(); + Vector128 bits = Vector128.Equals(d, Vector128.Create((byte)0x80).AsByte()); + int mask = Vector128_.MoveMask(bits); return mask != 0xFFFF; } }