diff --git a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs
index 760296c9d3..4492b297ce 100644
--- a/src/ImageSharp/Common/Helpers/Vector128Utilities.cs
+++ b/src/ImageSharp/Common/Helpers/Vector128Utilities.cs
@@ -1300,4 +1300,68 @@ internal static class Vector128_
// Narrow back to signed bytes
return Vector128.Narrow(diffLo, diffHi);
}
+
+ ///
+ /// Create mask from the most significant bit of each 8-bit element in , and store the result.
+ ///
+ ///
+ /// The vector containing packed 8-bit integers from which to create the mask.
+ ///
+ ///
+ /// A 16-bit integer mask where each bit corresponds to the most significant bit of each 8-bit element
+ /// in .
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static int MoveMask(Vector128 value)
+ {
+ if (Sse2.IsSupported)
+ {
+ return Sse2.MoveMask(value);
+ }
+
+ if (AdvSimd.IsSupported)
+ {
+ // https://stackoverflow.com/questions/11870910/sse-mm-movemask-epi8-equivalent-method-for-arm-neon
+ Vector128 powers = Vector128.Create(1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128);
+ Vector128 masked = value & powers;
+
+ Vector128 sum8 = AdvSimd.AddPairwiseWidening(masked);
+ Vector128 sum16 = AdvSimd.AddPairwiseWidening(sum8);
+ Vector128 sum32 = AdvSimd.AddPairwiseWidening(sum16);
+
+ // Extract lower 8 bits of each 64-bit lane
+ byte lo = sum32.AsByte().GetElement(0);
+ byte hi = sum32.AsByte().GetElement(8);
+
+ return (hi << 8) | lo;
+ }
+
+ {
+ // Step 1: isolate MSBs
+ Vector128 msbMask = Vector128.Create((byte)0x80);
+ Vector128 masked = value & msbMask;
+
+ // Step 2: shift each byte so MSB lands in bit position [0..15]
+ // i.e. convert: 0x80 → 1 << i
+ Vector128 bitShifts = Vector128.Create((ushort)1, 2, 4, 8, 16, 32, 64, 128);
+ Vector128 bitShiftsHigh = Vector128.Create(256, 512, 1024, 2048, 4096, 8192, 16384, 32768);
+
+ // Step 3: widen to ushort
+ (Vector128 lo, Vector128 hi) = Vector128.Widen(masked);
+
+ // Step 4: compare > 0 to get 0xFFFF where MSB was set
+ lo = Vector128.ConditionalSelect(Vector128.Equals(lo, Vector128.Zero), Vector128.Zero, bitShifts);
+ hi = Vector128.ConditionalSelect(Vector128.Equals(hi, Vector128.Zero), Vector128.Zero, bitShiftsHigh);
+
+ // Step 5: bitwise OR the two halves
+ Vector128 maskVector = lo | hi;
+
+ // Step 6: horizontal OR reduction via shuffles
+ maskVector |= Vector128.Shuffle(maskVector, Vector128.Create((ushort)4, 5, 6, 7, 0, 1, 2, 3));
+ maskVector |= Vector128.Shuffle(maskVector, Vector128.Create((ushort)2, 3, 0, 1, 6, 7, 4, 5));
+ maskVector |= Vector128.Shuffle(maskVector, Vector128.Create((ushort)1, 0, 3, 2, 5, 4, 7, 6));
+
+ return maskVector.ToScalar();
+ }
+ }
}
diff --git a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs
index 71dfadc399..e61dcb6bf9 100644
--- a/src/ImageSharp/Common/Helpers/Vector256Utilities.cs
+++ b/src/ImageSharp/Common/Helpers/Vector256Utilities.cs
@@ -231,6 +231,27 @@ internal static class Vector256_
return Vector256.Narrow(lefClamped, rightClamped);
}
+ ///
+ /// Packs signed 16-bit integers to signed 8-bit integers and saturates.
+ ///
+ /// The left hand source vector.
+ /// The right hand source vector.
+ /// The .
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector256 PackSignedSaturate(Vector256 left, Vector256 right)
+ {
+ if (Avx2.IsSupported)
+ {
+ return Avx2.PackSignedSaturate(left, right);
+ }
+
+ Vector256 min = Vector256.Create((short)sbyte.MinValue);
+ Vector256 max = Vector256.Create((short)sbyte.MaxValue);
+ Vector256 lefClamped = Clamp(left, min, max);
+ Vector256 rightClamped = Clamp(right, min, max);
+ return Vector256.Narrow(lefClamped, rightClamped);
+ }
+
///
/// Restricts a vector between a minimum and a maximum value.
///
@@ -466,6 +487,28 @@ internal static class Vector256_
Vector128_.SubtractSaturate(left.GetUpper(), right.GetUpper()));
}
+ ///
+ /// Create mask from the most significant bit of each 8-bit element in , and store the result.
+ ///
+ ///
+ /// The vector containing packed 8-bit integers from which to create the mask.
+ ///
+ ///
+ /// A 16-bit integer mask where each bit corresponds to the most significant bit of each 8-bit element
+ /// in .
+ ///
+ public static int MoveMask(Vector256 value)
+ {
+ if (Avx2.IsSupported)
+ {
+ return Avx2.MoveMask(value);
+ }
+
+ int loMask = Vector128_.MoveMask(value.GetLower());
+ int hiMask = Vector128_.MoveMask(value.GetUpper());
+ return loMask | (hiMask << 16);
+ }
+
[DoesNotReturn]
private static void ThrowUnreachableException() => throw new UnreachableException();
}
diff --git a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs
index 43dab1ffc4..c7ce12fc7b 100644
--- a/src/ImageSharp/Formats/Webp/AlphaDecoder.cs
+++ b/src/ImageSharp/Formats/Webp/AlphaDecoder.cs
@@ -6,7 +6,6 @@ using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.Formats.Webp.BitReader;
@@ -314,7 +313,7 @@ internal class AlphaDecoder : IDisposable
private static void HorizontalUnfilter(Span prev, Span input, Span dst, int width)
{
- if ((Sse2.IsSupported || AdvSimd.IsSupported) && width >= 9)
+ if (Vector128.IsHardwareAccelerated && width >= 9)
{
dst[0] = (byte)(input[0] + (prev.IsEmpty ? 0 : prev[0]));
nuint i;
@@ -362,7 +361,7 @@ internal class AlphaDecoder : IDisposable
{
HorizontalUnfilter(null, input, dst, width);
}
- else if (Avx2.IsSupported)
+ else if (Vector256.IsHardwareAccelerated)
{
ref byte inputRef = ref MemoryMarshal.GetReference(input);
ref byte prevRef = ref MemoryMarshal.GetReference(prev);
@@ -374,7 +373,7 @@ internal class AlphaDecoder : IDisposable
{
Vector256 a0 = Unsafe.As>(ref Unsafe.Add(ref inputRef, i));
Vector256 b0 = Unsafe.As>(ref Unsafe.Add(ref prevRef, i));
- Vector256 c0 = Avx2.Add(a0.AsByte(), b0.AsByte());
+ Vector256 c0 = a0.AsByte() + b0.AsByte();
ref byte outputRef = ref Unsafe.Add(ref dstRef, i);
Unsafe.As>(ref outputRef) = c0;
}
diff --git a/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs b/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs
index 1ca409f9a4..b08fe15f51 100644
--- a/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs
+++ b/src/ImageSharp/Formats/Webp/WebpCommonUtils.cs
@@ -3,7 +3,7 @@
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.X86;
+using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.PixelFormats;
namespace SixLabors.ImageSharp.Formats.Webp;
@@ -20,7 +20,7 @@ internal static class WebpCommonUtils
/// Returns true if alpha has non-0xff values.
public static unsafe bool CheckNonOpaque(ReadOnlySpan row)
{
- if (Avx2.IsSupported)
+ if (Vector256.IsHardwareAccelerated)
{
ReadOnlySpan rowBytes = MemoryMarshal.AsBytes(row);
int i = 0;
@@ -32,19 +32,19 @@ internal static class WebpCommonUtils
for (; i + 128 <= length; i += 128)
{
- Vector256 a0 = Avx.LoadVector256(src + i).AsByte();
- Vector256 a1 = Avx.LoadVector256(src + i + 32).AsByte();
- Vector256 a2 = Avx.LoadVector256(src + i + 64).AsByte();
- Vector256 a3 = Avx.LoadVector256(src + i + 96).AsByte();
- Vector256 b0 = Avx2.And(a0, alphaMaskVector256).AsInt32();
- Vector256 b1 = Avx2.And(a1, alphaMaskVector256).AsInt32();
- Vector256 b2 = Avx2.And(a2, alphaMaskVector256).AsInt32();
- Vector256 b3 = Avx2.And(a3, alphaMaskVector256).AsInt32();
- Vector256 c0 = Avx2.PackSignedSaturate(b0, b1).AsInt16();
- Vector256 c1 = Avx2.PackSignedSaturate(b2, b3).AsInt16();
- Vector256 d = Avx2.PackSignedSaturate(c0, c1).AsByte();
- Vector256 bits = Avx2.CompareEqual(d, all0x80Vector256);
- int mask = Avx2.MoveMask(bits);
+ Vector256 a0 = Vector256.Load(src + i).AsByte();
+ Vector256 a1 = Vector256.Load(src + i + 32).AsByte();
+ Vector256 a2 = Vector256.Load(src + i + 64).AsByte();
+ Vector256 a3 = Vector256.Load(src + i + 96).AsByte();
+ Vector256 b0 = (a0 & alphaMaskVector256).AsInt32();
+ Vector256 b1 = (a1 & alphaMaskVector256).AsInt32();
+ Vector256 b2 = (a2 & alphaMaskVector256).AsInt32();
+ Vector256 b3 = (a3 & alphaMaskVector256).AsInt32();
+ Vector256 c0 = Vector256_.PackSignedSaturate(b0, b1).AsInt16();
+ Vector256 c1 = Vector256_.PackSignedSaturate(b2, b3).AsInt16();
+ Vector256 d = Vector256_.PackSignedSaturate(c0, c1).AsByte();
+ Vector256 bits = Vector256.Equals(d, all0x80Vector256);
+ int mask = Vector256_.MoveMask(bits);
if (mask != -1)
{
return true;
@@ -53,7 +53,7 @@ internal static class WebpCommonUtils
for (; i + 64 <= length; i += 64)
{
- if (IsNoneOpaque64Bytes(src, i))
+ if (IsNoneOpaque64BytesVector128(src, i))
{
return true;
}
@@ -61,7 +61,7 @@ internal static class WebpCommonUtils
for (; i + 32 <= length; i += 32)
{
- if (IsNoneOpaque32Bytes(src, i))
+ if (IsNonOpaque32BytesVector128(src, i))
{
return true;
}
@@ -76,7 +76,7 @@ internal static class WebpCommonUtils
}
}
}
- else if (Sse2.IsSupported)
+ else if (Vector128.IsHardwareAccelerated)
{
ReadOnlySpan rowBytes = MemoryMarshal.AsBytes(row);
int i = 0;
@@ -85,7 +85,7 @@ internal static class WebpCommonUtils
{
for (; i + 64 <= length; i += 64)
{
- if (IsNoneOpaque64Bytes(src, i))
+ if (IsNoneOpaque64BytesVector128(src, i))
{
return true;
}
@@ -93,7 +93,7 @@ internal static class WebpCommonUtils
for (; i + 32 <= length; i += 32)
{
- if (IsNoneOpaque32Bytes(src, i))
+ if (IsNonOpaque32BytesVector128(src, i))
{
return true;
}
@@ -122,38 +122,38 @@ internal static class WebpCommonUtils
return false;
}
- private static unsafe bool IsNoneOpaque64Bytes(byte* src, int i)
+ private static unsafe bool IsNoneOpaque64BytesVector128(byte* src, int i)
{
Vector128 alphaMask = Vector128.Create(0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255);
- Vector128 a0 = Sse2.LoadVector128(src + i).AsByte();
- Vector128 a1 = Sse2.LoadVector128(src + i + 16).AsByte();
- Vector128 a2 = Sse2.LoadVector128(src + i + 32).AsByte();
- Vector128 a3 = Sse2.LoadVector128(src + i + 48).AsByte();
- Vector128 b0 = Sse2.And(a0, alphaMask).AsInt32();
- Vector128 b1 = Sse2.And(a1, alphaMask).AsInt32();
- Vector128 b2 = Sse2.And(a2, alphaMask).AsInt32();
- Vector128 b3 = Sse2.And(a3, alphaMask).AsInt32();
- Vector128 c0 = Sse2.PackSignedSaturate(b0, b1).AsInt16();
- Vector128 c1 = Sse2.PackSignedSaturate(b2, b3).AsInt16();
- Vector128 d = Sse2.PackSignedSaturate(c0, c1).AsByte();
- Vector128 bits = Sse2.CompareEqual(d, Vector128.Create((byte)0x80).AsByte());
- int mask = Sse2.MoveMask(bits);
+ Vector128 a0 = Vector128.Load(src + i).AsByte();
+ Vector128 a1 = Vector128.Load(src + i + 16).AsByte();
+ Vector128 a2 = Vector128.Load(src + i + 32).AsByte();
+ Vector128 a3 = Vector128.Load(src + i + 48).AsByte();
+ Vector128 b0 = (a0 & alphaMask).AsInt32();
+ Vector128 b1 = (a1 & alphaMask).AsInt32();
+ Vector128 b2 = (a2 & alphaMask).AsInt32();
+ Vector128 b3 = (a3 & alphaMask).AsInt32();
+ Vector128 c0 = Vector128_.PackSignedSaturate(b0, b1).AsInt16();
+ Vector128 c1 = Vector128_.PackSignedSaturate(b2, b3).AsInt16();
+ Vector128 d = Vector128_.PackSignedSaturate(c0, c1).AsByte();
+ Vector128 bits = Vector128.Equals(d, Vector128.Create((byte)0x80).AsByte());
+ int mask = Vector128_.MoveMask(bits);
return mask != 0xFFFF;
}
- private static unsafe bool IsNoneOpaque32Bytes(byte* src, int i)
+ private static unsafe bool IsNonOpaque32BytesVector128(byte* src, int i)
{
Vector128 alphaMask = Vector128.Create(0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255);
- Vector128 a0 = Sse2.LoadVector128(src + i).AsByte();
- Vector128 a1 = Sse2.LoadVector128(src + i + 16).AsByte();
- Vector128 b0 = Sse2.And(a0, alphaMask).AsInt32();
- Vector128 b1 = Sse2.And(a1, alphaMask).AsInt32();
- Vector128 c = Sse2.PackSignedSaturate(b0, b1).AsInt16();
- Vector128 d = Sse2.PackSignedSaturate(c, c).AsByte();
- Vector128 bits = Sse2.CompareEqual(d, Vector128.Create((byte)0x80).AsByte());
- int mask = Sse2.MoveMask(bits);
+ Vector128 a0 = Vector128.Load(src + i).AsByte();
+ Vector128 a1 = Vector128.Load(src + i + 16).AsByte();
+ Vector128 b0 = (a0 & alphaMask).AsInt32();
+ Vector128 b1 = (a1 & alphaMask).AsInt32();
+ Vector128 c = Vector128_.PackSignedSaturate(b0, b1).AsInt16();
+ Vector128 d = Vector128_.PackSignedSaturate(c, c).AsByte();
+ Vector128 bits = Vector128.Equals(d, Vector128.Create((byte)0x80).AsByte());
+ int mask = Vector128_.MoveMask(bits);
return mask != 0xFFFF;
}
}