diff --git a/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs b/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs index 4756dea867..f465741367 100644 --- a/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs +++ b/src/ImageSharp/Formats/Webp/Lossy/LossyUtils.cs @@ -5,6 +5,7 @@ using System.Buffers.Binary; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; // ReSharper disable InconsistentNaming @@ -14,7 +15,7 @@ internal static class LossyUtils { // Note: method name in libwebp reference implementation is called VP8SSE16x16. [MethodImpl(InliningOptions.ShortMethod)] - public static int Vp8_Sse16X16(Span a, Span b) + public static int Vp8_Sse16x16(Span a, Span b) { if (Avx2.IsSupported) { @@ -26,12 +27,17 @@ internal static class LossyUtils return Vp8_Sse16xN_Sse2(a, b, 8); } + if (AdvSimd.IsSupported) + { + return Vp8_Sse16x16_Neon(a, b); + } + return Vp8_SseNxN(a, b, 16, 16); } // Note: method name in libwebp reference implementation is called VP8SSE16x8. [MethodImpl(InliningOptions.ShortMethod)] - public static int Vp8_Sse16X8(Span a, Span b) + public static int Vp8_Sse16x8(Span a, Span b) { if (Avx2.IsSupported) { @@ -43,12 +49,17 @@ internal static class LossyUtils return Vp8_Sse16xN_Sse2(a, b, 4); } + if (AdvSimd.IsSupported) + { + return Vp8_Sse16x8_Neon(a, b); + } + return Vp8_SseNxN(a, b, 16, 8); } // Note: method name in libwebp reference implementation is called VP8SSE4x4. [MethodImpl(InliningOptions.ShortMethod)] - public static int Vp8_Sse4X4(Span a, Span b) + public static int Vp8_Sse4x4(Span a, Span b) { if (Avx2.IsSupported) { @@ -119,6 +130,11 @@ internal static class LossyUtils return Numerics.ReduceSum(sum); } + if (AdvSimd.IsSupported) + { + return Vp8_Sse4x4_Neon(a, b); + } + return Vp8_SseNxN(a, b, 4, 4); } @@ -201,6 +217,91 @@ internal static class LossyUtils return Numerics.ReduceSum(sum); } + [MethodImpl(InliningOptions.ShortMethod)] + private static int Vp8_Sse16x16_Neon(Span a, Span b) + { + Vector128 sum = Vector128.Zero; + for (int y = 0; y < 16; y++) + { + sum = AccumulateSSE16Neon(a.Slice(y * WebpConstants.Bps), b.Slice(y * WebpConstants.Bps), sum); + } + + return ReduceSum(sum); + } + + [MethodImpl(InliningOptions.ShortMethod)] + private static int Vp8_Sse16x8_Neon(Span a, Span b) + { + Vector128 sum = Vector128.Zero; + for (int y = 0; y < 8; y++) + { + sum = AccumulateSSE16Neon(a.Slice(y * WebpConstants.Bps), b.Slice(y * WebpConstants.Bps), sum); + } + + return ReduceSum(sum); + } + + [MethodImpl(InliningOptions.ShortMethod)] + private static int Vp8_Sse4x4_Neon(Span a, Span b) + { + Vector128 a0 = Load4x4Neon(a).AsByte(); + Vector128 b0 = Load4x4Neon(b).AsByte(); + Vector128 absDiff = AdvSimd.AbsoluteDifference(a0, b0); + Vector64 absDiffLower = absDiff.GetLower().AsByte(); + Vector64 absDiffUpper = absDiff.GetUpper().AsByte(); + Vector128 prod1 = AdvSimd.MultiplyWideningLower(absDiffLower, absDiffLower); + Vector128 prod2 = AdvSimd.MultiplyWideningLower(absDiffUpper, absDiffUpper); + + // pair-wise adds and widen. + Vector128 sum1 = AdvSimd.AddPairwiseWidening(prod1); + Vector128 sum2 = AdvSimd.AddPairwiseWidening(prod2); + return ReduceSum(AdvSimd.Add(sum1, sum2)); + } + + // Load all 4x4 pixels into a single Vector128 + [MethodImpl(InliningOptions.ShortMethod)] + private static unsafe Vector128 Load4x4Neon(Span src) + { + fixed (byte* srcRef = &MemoryMarshal.GetReference(src)) + { + Vector128 output = Vector128.Zero; + output = AdvSimd.LoadAndInsertScalar(output, 0, (uint*)srcRef); + output = AdvSimd.LoadAndInsertScalar(output, 1, (uint*)(srcRef + WebpConstants.Bps)); + output = AdvSimd.LoadAndInsertScalar(output, 2, (uint*)(srcRef + (WebpConstants.Bps * 2))); + output = AdvSimd.LoadAndInsertScalar(output, 3, (uint*)(srcRef + (WebpConstants.Bps * 3))); + return output; + } + } + + [MethodImpl(InliningOptions.ShortMethod)] + private static int ReduceSum(Vector128 sum) + { + Vector128 sum2 = AdvSimd.AddPairwiseWidening(sum); + Vector64 sum3 = AdvSimd.Add(sum2.GetLower().AsUInt32(), sum2.GetUpper().AsUInt32()); + return (int)AdvSimd.Extract(sum3, 0); + } + + [MethodImpl(InliningOptions.ShortMethod)] + private static Vector128 AccumulateSSE16Neon(Span a, Span b, Vector128 sum) + { + ref byte aRef = ref MemoryMarshal.GetReference(a); + ref byte bRef = ref MemoryMarshal.GetReference(b); + + Vector128 a0 = Unsafe.As>(ref aRef); + Vector128 b0 = Unsafe.As>(ref bRef); + + Vector128 absDiff = AdvSimd.AbsoluteDifference(a0, b0); + Vector64 absDiffLower = absDiff.GetLower(); + Vector64 absDiffUpper = absDiff.GetUpper(); + Vector128 prod1 = AdvSimd.MultiplyWideningLower(absDiffLower, absDiffLower); + Vector128 prod2 = AdvSimd.MultiplyWideningLower(absDiffUpper, absDiffUpper); + + // pair-wise adds and widen. + Vector128 sum1 = AdvSimd.AddPairwiseWidening(prod1); + Vector128 sum2 = AdvSimd.AddPairwiseWidening(prod2); + return AdvSimd.Add(sum, AdvSimd.Add(sum1, sum2)); + } + [MethodImpl(InliningOptions.ShortMethod)] private static Vector128 SubtractAndAccumulate(Vector128 a, Vector128 b) { diff --git a/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs b/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs index fed9c16d4d..fca0d03f23 100644 --- a/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs +++ b/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs @@ -53,7 +53,7 @@ internal static unsafe class QuantEnc rdCur.Nz = (uint)ReconstructIntra16(it, dqm, rdCur, tmpDst, mode); // Measure RD-score. - rdCur.D = LossyUtils.Vp8_Sse16X16(src, tmpDst); + rdCur.D = LossyUtils.Vp8_Sse16x16(src, tmpDst); rdCur.SD = tlambda != 0 ? Mult8B(tlambda, LossyUtils.Vp8Disto16X16(src, tmpDst, WeightY, scratch)) : 0; rdCur.H = WebpConstants.Vp8FixedCostsI16[mode]; rdCur.R = it.GetCostLuma16(rdCur, proba, res); @@ -145,7 +145,7 @@ internal static unsafe class QuantEnc rdTmp.Nz = (uint)ReconstructIntra4(it, dqm, tmpLevels, src, tmpDst, mode); // Compute RD-score. - rdTmp.D = LossyUtils.Vp8_Sse4X4(src, tmpDst); + rdTmp.D = LossyUtils.Vp8_Sse4x4(src, tmpDst); rdTmp.SD = tlambda != 0 ? Mult8B(tlambda, LossyUtils.Vp8Disto4X4(src, tmpDst, WeightY, scratch)) : 0; rdTmp.H = modeCosts[mode]; @@ -235,7 +235,7 @@ internal static unsafe class QuantEnc rdUv.Nz = (uint)ReconstructUv(it, dqm, rdUv, tmpDst, mode); // Compute RD-score - rdUv.D = LossyUtils.Vp8_Sse16X8(src, tmpDst); + rdUv.D = LossyUtils.Vp8_Sse16x8(src, tmpDst); rdUv.SD = 0; // not calling TDisto here: it tends to flatten areas. rdUv.H = WebpConstants.Vp8FixedCostsUv[mode]; rdUv.R = it.GetCostUv(rdUv, proba, res); @@ -389,7 +389,7 @@ internal static unsafe class QuantEnc for (mode = 0; mode < WebpConstants.NumPredModes; ++mode) { Span reference = it.YuvP.AsSpan(Vp8Encoding.Vp8I16ModeOffsets[mode]); - long score = (LossyUtils.Vp8_Sse16X16(src, reference) * WebpConstants.RdDistoMult) + (WebpConstants.Vp8FixedCostsI16[mode] * lambdaDi16); + long score = (LossyUtils.Vp8_Sse16x16(src, reference) * WebpConstants.RdDistoMult) + (WebpConstants.Vp8FixedCostsI16[mode] * lambdaDi16); if (mode > 0 && WebpConstants.Vp8FixedCostsI16[mode] > bitLimit) { @@ -436,7 +436,7 @@ internal static unsafe class QuantEnc for (mode = 0; mode < WebpConstants.NumBModes; ++mode) { Span reference = it.YuvP.AsSpan(Vp8Encoding.Vp8I4ModeOffsets[mode]); - long score = (LossyUtils.Vp8_Sse4X4(src, reference) * WebpConstants.RdDistoMult) + (modeCosts[mode] * lambdaDi4); + long score = (LossyUtils.Vp8_Sse4x4(src, reference) * WebpConstants.RdDistoMult) + (modeCosts[mode] * lambdaDi4); if (score < bestI4Score) { bestI4Mode = mode; @@ -485,7 +485,7 @@ internal static unsafe class QuantEnc for (mode = 0; mode < WebpConstants.NumPredModes; ++mode) { Span reference = it.YuvP.AsSpan(Vp8Encoding.Vp8UvModeOffsets[mode]); - long score = (LossyUtils.Vp8_Sse16X8(src, reference) * WebpConstants.RdDistoMult) + (WebpConstants.Vp8FixedCostsUv[mode] * lambdaDuv); + long score = (LossyUtils.Vp8_Sse16x8(src, reference) * WebpConstants.RdDistoMult) + (WebpConstants.Vp8FixedCostsUv[mode] * lambdaDuv); if (score < bestUvScore) { bestMode = mode; diff --git a/tests/ImageSharp.Tests/Formats/WebP/LossyUtilsTests.cs b/tests/ImageSharp.Tests/Formats/WebP/LossyUtilsTests.cs index 69b503b5e5..4682536d05 100644 --- a/tests/ImageSharp.Tests/Formats/WebP/LossyUtilsTests.cs +++ b/tests/ImageSharp.Tests/Formats/WebP/LossyUtilsTests.cs @@ -140,7 +140,7 @@ public class LossyUtilsTests int expected = 2063; // act - int actual = LossyUtils.Vp8_Sse16X16(a, b); + int actual = LossyUtils.Vp8_Sse16x16(a, b); // assert Assert.Equal(expected, actual); @@ -186,7 +186,7 @@ public class LossyUtilsTests int expected = 749; // act - int actual = LossyUtils.Vp8_Sse16X8(a, b); + int actual = LossyUtils.Vp8_Sse16x8(a, b); // assert Assert.Equal(expected, actual); @@ -218,7 +218,7 @@ public class LossyUtilsTests int expected = 27; // act - int actual = LossyUtils.Vp8_Sse4X4(a, b); + int actual = LossyUtils.Vp8_Sse4x4(a, b); // assert Assert.Equal(expected, actual);