diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.Operations.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.Operations.cs index c2b3c9023..30bcdd9fa 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.Operations.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.Operations.cs @@ -17,7 +17,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration; internal static partial class Av1SelfGuidedFilter { /// - /// Applies self-guided restoration with the AVX2 traversal used by libaom. + /// Applies self-guided restoration with the 256-bit traversal. /// /// The bordered processing-unit source rectangle. /// The number of samples between source rows. @@ -216,7 +216,7 @@ internal static partial class Av1SelfGuidedFilter } /// - /// Builds the summed-area tables consumed by the AVX2 coefficient stage. + /// Builds the summed-area tables consumed by the 256-bit coefficient stage. /// /// The complete bordered source rectangle. /// The number of samples between source rows. @@ -258,7 +258,7 @@ internal static partial class Av1SelfGuidedFilter // Eight packed 16-bit samples become eight 32-bit lanes. The prefix scans mirror // libaom's scan_32, and the replicated carry joins consecutive vector batches. Vector128 packed = Vector128.LoadUnsafe(ref sourceBase, (nuint)(sourceRowOffset + column)); - Vector256 samples = Avx2.ConvertToVector256Int32(packed); + Vector256 samples = Vector256.WidenLower(Vector256.Create(packed, Vector128.Zero)).AsInt32(); Vector256 squares = samples * samples; Vector256 scannedSums = Scan(samples); Vector256 scannedSquares = Scan(squares); @@ -366,12 +366,21 @@ internal static partial class Av1SelfGuidedFilter [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 Scan(Vector256 values) { - // AVX2 byte shifts operate independently on the two 128-bit halves. After the two - // within-half scans, the lower-half total is added to every lane of the upper half. - Vector256 scan = values + Avx2.ShiftLeftLogical128BitLane(values.AsByte(), sizeof(int)).AsInt32(); - scan += Avx2.ShiftLeftLogical128BitLane(scan.AsByte(), sizeof(int) * 2).AsInt32(); - Vector256 lowerTotal = Vector256.Create(Vector128.Zero, Vector128.Create(scan.GetElement(3))); - return scan + lowerTotal; + if (Avx2.IsSupported) + { + // AVX2 lane shifts provide the shortest x86 dependency chain. After scanning each 128-bit half, the lower + // half total is broadcast into the upper half so the result remains one continuous eight-lane prefix. + Vector256 avx2Scan = values + Avx2.ShiftLeftLogical128BitLane(values.AsByte(), sizeof(int)).AsInt32(); + avx2Scan += Avx2.ShiftLeftLogical128BitLane(avx2Scan.AsByte(), sizeof(int) * 2).AsInt32(); + Vector256 lowerTotal = Vector256.Create(Vector128.Zero, Vector128.Create(avx2Scan.GetElement(3))); + return avx2Scan + lowerTotal; + } + + // Each shuffle shifts the preceding partial sums by one, two, and four lanes. The portable shuffle is required + // because indices outside the vector produce zero; ShuffleNative is allowed to wrap those indices on some ISAs. + Vector256 scan = values + Vector256.Shuffle(values, Vector256.Create(8, 0, 1, 2, 3, 4, 5, 6)); + scan += Vector256.Shuffle(scan, Vector256.Create(8, 8, 0, 1, 2, 3, 4, 5)); + return scan + Vector256.Shuffle(scan, Vector256.Create(8, 8, 8, 8, 0, 1, 2, 3)); } /// @@ -389,7 +398,7 @@ internal static partial class Av1SelfGuidedFilter } /// - /// Calculates the coefficient grid in eight-sample AVX2 batches. + /// Calculates the coefficient grid in eight-sample SIMD batches. /// /// The processing-unit width in samples. /// The processing-unit height in samples. @@ -682,12 +691,29 @@ internal static partial class Av1SelfGuidedFilter [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe Vector256 LookupBlendFactors(Vector256 indices) { - // Variance normalization bounds every index to the 256-entry table. AVX2 gather keeps the eight independent - // column lookups in the vector pipeline instead of materializing an intermediate scalar scale buffer. - fixed (int* table = XByXPlusOne) + ReadOnlySpan table = XByXPlusOne; + + if (Avx2.IsSupported) { - return Avx2.GatherVector256(table, indices.AsInt32(), sizeof(int)); + // Variance normalization bounds every index to the 256-entry table. AVX2 gather keeps all eight independent + // column lookups in the vector pipeline instead of materializing an intermediate scalar scale buffer. + fixed (int* tablePointer = table) + { + return Avx2.GatherVector256(tablePointer, indices.AsInt32(), sizeof(int)); + } } + + // Vector256 has no portable indexed-load operation. Constructing the result from eight bounded reads retains + // the 256-bit coefficient pipeline on other implementations without allocating or adding another row pass. + return Vector256.Create( + table[(int)indices.GetElement(0)], + table[(int)indices.GetElement(1)], + table[(int)indices.GetElement(2)], + table[(int)indices.GetElement(3)], + table[(int)indices.GetElement(4)], + table[(int)indices.GetElement(5)], + table[(int)indices.GetElement(6)], + table[(int)indices.GetElement(7)]); } /// @@ -767,7 +793,7 @@ internal static partial class Av1SelfGuidedFilter } /// - /// Produces the radius-two filtered values in eight-sample AVX2 batches. + /// Produces the radius-two filtered values in eight-sample SIMD batches. /// /// The bordered processing-unit source rectangle. /// The number of samples between source rows. @@ -810,7 +836,7 @@ internal static partial class Av1SelfGuidedFilter Vector256 factors = CrossSum(blendFactors, coefficientRowOffset + column, bufferStride, row, vector); Vector256 means = CrossSum(localMeans, coefficientRowOffset + column, bufferStride, row, vector); Vector128 packed = Vector128.LoadUnsafe(ref sourceBase, (nuint)(sourceRowOffset + column)); - Vector256 samples = Avx2.ConvertToVector256Int32(packed); + Vector256 samples = Vector256.WidenLower(Vector256.Create(packed, Vector128.Zero)).AsInt32(); Vector256 values = Vector256.ShiftRightArithmetic((factors * samples) + means + rounding, roundingBits); values.StoreUnsafe(ref filteredBase, (nuint)(filteredRowOffset + column)); } @@ -886,7 +912,7 @@ internal static partial class Av1SelfGuidedFilter } /// - /// Produces the radius-one filtered values in eight-sample AVX2 batches. + /// Produces the radius-one filtered values in eight-sample SIMD batches. /// /// The bordered processing-unit source rectangle. /// The number of samples between source rows. @@ -926,7 +952,7 @@ internal static partial class Av1SelfGuidedFilter Vector256 factors = CrossSum(blendFactors, coefficientRowOffset + column, bufferStride, vector); Vector256 means = CrossSum(localMeans, coefficientRowOffset + column, bufferStride, vector); Vector128 packed = Vector128.LoadUnsafe(ref sourceBase, (nuint)(sourceRowOffset + column)); - Vector256 samples = Avx2.ConvertToVector256Int32(packed); + Vector256 samples = Vector256.WidenLower(Vector256.Create(packed, Vector128.Zero)).AsInt32(); Vector256 values = Vector256.ShiftRightArithmetic((factors * samples) + means + rounding, roundingBits); values.StoreUnsafe(ref filteredBase, (nuint)(filteredRowOffset + column)); } @@ -1183,7 +1209,7 @@ internal static partial class Av1SelfGuidedFilter } /// - /// Projects the two restored signals in eight-sample AVX2 batches. + /// Projects the two restored signals in eight-sample SIMD batches. /// /// The bordered processing-unit source rectangle. /// The number of samples between source rows. @@ -1233,7 +1259,7 @@ internal static partial class Av1SelfGuidedFilter for (; column <= vectorEnd; column += Vector256.Count) { Vector128 packed = Vector128.LoadUnsafe(ref sourceBase, (nuint)(sourceRowOffset + column)); - Vector256 samples = Avx2.ConvertToVector256Int32(packed); + Vector256 samples = Vector256.WidenLower(Vector256.Create(packed, Vector128.Zero)).AsInt32(); Vector256 unfiltered = Vector256.ShiftLeft(samples, RestorationBits); Vector256 projected = Vector256.ShiftLeft(unfiltered, ProjectionBits); if (radii[0] > 0) @@ -1251,8 +1277,8 @@ internal static partial class Av1SelfGuidedFilter Vector256 result = Vector256.ShiftRightArithmetic(projected + rounding, projectionShift); result = Vector256.Min(Vector256.Max(result, Vector256.Zero), maximumSample); - // Narrowing the result with a zero upper vector places the eight ordered samples - // in the lower 128 bits, which can be stored without the AVX2 pack permutation. + // Narrowing the result with a zero upper vector places the eight ordered samples in the lower 128 bits, + // which can be stored directly without an ISA-specific lane permutation. Vector128 narrowed = Vector256.Narrow(result.AsUInt32(), Vector256.Zero).GetLower(); narrowed.StoreUnsafe(ref destinationBase, (nuint)(destinationRowOffset + column)); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.cs index 54b3d7899..bb76c805d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.cs @@ -3,7 +3,6 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration; @@ -12,9 +11,9 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration; /// /// /// The accelerated paths process adjacent output columns in SIMD lanes and use the caller-provided scratch span for -/// filtered samples, local coefficients, and padded integral images. AVX2 is selected where variance-to-blend lookup -/// can use a native gather; the portable 128-bit path performs four scalar table reads while retaining vectorized -/// window and projection arithmetic. The scalar path uses the same fixed-point units and scratch partition. +/// filtered samples, local coefficients, and padded integral images. The 256-bit and 128-bit paths use portable +/// vector operations, with AVX2 selected locally for the prefix scan and variance-to-blend lookup when available. +/// The scalar path uses the same fixed-point units and scratch partition. /// internal static partial class Av1SelfGuidedFilter { @@ -157,9 +156,9 @@ internal static partial class Av1SelfGuidedFilter ReadOnlySpan projectionCoefficients, Span scratch) { - // The closed vector overloads share the same scratch layout and fixed-point equations. AVX2 is preferred here - // because its coefficient stage can gather four noncontiguous entries from the 256-value blend table. - if (Avx2.IsSupported) + // The closed vector overloads share the same scratch layout and fixed-point equations. Dispatch is based on + // portable vector width; ISA-specific acceleration is confined to the individual operation that requires it. + if (Vector256.IsHardwareAccelerated) { FilterBlock( source, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SelfGuidedFilterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SelfGuidedFilterTests.cs index b6cab2849..fc882d89a 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SelfGuidedFilterTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SelfGuidedFilterTests.cs @@ -28,10 +28,10 @@ public class Av1SelfGuidedFilterTests private const int ProjectionBits = 7; /// - /// The hardware configurations required to exercise the AVX2 path, portable 128-bit path, and scalar fallback. + /// The hardware configurations required to exercise AVX2-assisted 256-bit, portable 256-bit, 128-bit, and scalar execution. /// private const HwIntrinsics Configurations = - HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX2 | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; /// /// Gets the radii selected by each of the sixteen normative parameter sets.