diff --git a/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs b/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs index 2fcea8ceea..125645c7d0 100644 --- a/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs +++ b/src/ImageSharp/Formats/Webp/Lossy/QuantEnc.cs @@ -25,6 +25,12 @@ namespace SixLabors.ImageSharp.Formats.Webp.Lossy #if SUPPORTS_RUNTIME_INTRINSICS private static readonly Vector128 MaxCoeff2047 = Vector128.Create((short)MaxLevel); + private static readonly Vector256 MaxCoeff2047Vec256 = Vector256.Create((short)MaxLevel); + + private static readonly Vector256 Cst256 = Vector256.Create(0, 1, 2, 3, 8, 9, 254, 255, 10, 11, 4, 5, 6, 7, 12, 13, 2, 3, 8, 9, 10, 11, 4, 5, 254, 255, 6, 7, 12, 13, 14, 15); + + private static readonly Vector256 Cst78 = Vector256.Create(254, 255, 254, 255, 254, 255, 254, 255, 14, 15, 254, 255, 254, 255, 254, 255, 254, 255, 254, 255, 254, 255, 0, 1, 254, 255, 254, 255, 254, 255, 254, 255); + private static readonly Vector128 CstLo = Vector128.Create(0, 1, 2, 3, 8, 9, 254, 255, 10, 11, 4, 5, 6, 7, 12, 13); private static readonly Vector128 Cst7 = Vector128.Create(254, 255, 254, 255, 254, 255, 254, 255, 14, 15, 254, 255, 254, 255, 254, 255); @@ -531,7 +537,70 @@ namespace SixLabors.ImageSharp.Formats.Webp.Lossy public static int QuantizeBlock(Span input, Span output, ref Vp8Matrix mtx) { #if SUPPORTS_RUNTIME_INTRINSICS - if (Sse41.IsSupported) + if (Avx2.IsSupported) + { + // Load all inputs. + Vector256 input0 = Unsafe.As>(ref MemoryMarshal.GetReference(input)); + Vector256 iq0 = Unsafe.As>(ref mtx.IQ[0]); + Vector256 q0 = Unsafe.As>(ref mtx.Q[0]); + + // coeff = abs(in) + Vector256 coeff0 = Avx2.Abs(input0); + + // coeff = abs(in) + sharpen + Vector256 sharpen0 = Unsafe.As>(ref mtx.Sharpen[0]); + Avx2.Add(coeff0.AsInt16(), sharpen0); + + // out = (coeff * iQ + B) >> QFIX + // doing calculations with 32b precision (QFIX=17) + // out = (coeff * iQ) + Vector256 coeffiQ0H = Avx2.MultiplyHigh(coeff0, iq0); + Vector256 coeffiQ0L = Avx2.MultiplyLow(coeff0, iq0); + Vector256 out00 = Avx2.UnpackLow(coeffiQ0L, coeffiQ0H); + Vector256 out08 = Avx2.UnpackHigh(coeffiQ0L, coeffiQ0H); + + // out = (coeff * iQ + B) + Vector256 bias00 = Unsafe.As>(ref mtx.Bias[0]); + Vector256 bias08 = Unsafe.As>(ref mtx.Bias[8]); + out00 = Avx2.Add(out00.AsInt32(), bias00.AsInt32()).AsUInt16(); + out08 = Avx2.Add(out08.AsInt32(), bias08.AsInt32()).AsUInt16(); + + // out = QUANTDIV(coeff, iQ, B, QFIX) + out00 = Avx2.ShiftRightArithmetic(out00.AsInt32(), WebpConstants.QFix).AsUInt16(); + out08 = Avx2.ShiftRightArithmetic(out08.AsInt32(), WebpConstants.QFix).AsUInt16(); + + // Pack result as 16b. + Vector256 out0 = Avx2.PackSignedSaturate(out00.AsInt32(), out08.AsInt32()); + + // if (coeff > 2047) coeff = 2047 + out0 = Avx2.Min(out0, MaxCoeff2047Vec256); + + // Put the sign back. + out0 = Avx2.Sign(out0, input0); + + // in = out * Q + input0 = Avx2.MultiplyLow(out0, q0.AsInt16()); + ref short inputRef = ref MemoryMarshal.GetReference(input); + Unsafe.As>(ref inputRef) = input0; + + // zigzag the output before storing it. + Vector256 tmp256 = Avx2.Shuffle(out0.AsByte(), Cst256); + Vector256 tmp78 = Avx2.Shuffle(out0.AsByte(), Cst78); + + // Reverse the order of the 16-byte lanes. + Vector256 tmp87 = Avx2.Permute2x128(tmp78, tmp78, 1); + Vector256 outZ = Avx2.Or(tmp256, tmp87).AsInt16(); + + ref short outputRef = ref MemoryMarshal.GetReference(output); + Unsafe.As>(ref outputRef) = outZ; + + Vector256 packedOutput = Avx2.PackSignedSaturate(outZ, outZ); + + // Detect if all 'out' values are zeros or not. + Vector256 cmpeq = Avx2.CompareEqual(packedOutput, Vector256.Zero); + return Avx2.MoveMask(cmpeq) != -1 ? 1 : 0; + } + else if (Sse41.IsSupported) { // Load all inputs. Vector128 input0 = Unsafe.As>(ref MemoryMarshal.GetReference(input)); @@ -579,7 +648,7 @@ namespace SixLabors.ImageSharp.Formats.Webp.Lossy out08 = Sse2.ShiftRightArithmetic(out08.AsInt32(), WebpConstants.QFix).AsUInt16(); out12 = Sse2.ShiftRightArithmetic(out12.AsInt32(), WebpConstants.QFix).AsUInt16(); - // pack result as 16b + // Pack result as 16b. Vector128 out0 = Sse2.PackSignedSaturate(out00.AsInt32(), out04.AsInt32()); Vector128 out8 = Sse2.PackSignedSaturate(out08.AsInt32(), out12.AsInt32()); @@ -587,7 +656,7 @@ namespace SixLabors.ImageSharp.Formats.Webp.Lossy out0 = Sse2.Min(out0, MaxCoeff2047); out8 = Sse2.Min(out8, MaxCoeff2047); - // put sign back + // Put the sign back. out0 = Ssse3.Sign(out0, input0); out8 = Ssse3.Sign(out8, input8);