diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 08f150746..67724fe06 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -268,7 +268,7 @@ This snapshot pins or classifies the available references and failures; it does | `Av1PredictionDecoder`, `Av1IntraPredictor`, `Av1ChromaFromLumaContext`, `Av1ChromaFromLumaPredictor`, `Av1PartitionInfo`, and the DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictor operators | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/blockd.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design the shared predictor contract, sample layout, reference preparation, and scratch ownership for widest-to-narrowest SIMD traversal, then provide the equivalent scalar fallback through that contract. Preserve luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | | `Av1TileReader` palette mode/color-map parsing, `Av1SymbolDecoder` palette distributions, `Av1BlockModeInfo` palette state, and `Av1PredictionDecoder` palette reconstruction | AV1 sections 5.11.46, 5.11.49, and 7.11.2 palette prediction | libaom `av1/decoder/decodemv.c`, `av1/decoder/detokenize.c`, `av1/decoder/decoder.h`, `av1/common/pred_common.c`, `av1/common/pred_common.h`, and `av1/common/entropymode.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Reuse the existing tile range decoder and frame-owned mode map while porting the normative palette cache merge, high-bit-depth color deltas, tile-adaptive mode/size/index distributions, diagonal color-map traversal, edge padding, and direct palette-sample reconstruction. This is AV1 still-image compression syntax and does not add retained video reference state or any ISO BMFF surface. | | `Av1DeblockingFilter`, `Av1LoopFilterContext`, and `Av1LoopFilterDecoder` | AV1 section 7.14 deblocking loop filter | libaom `aom_dsp/loopfilter.c` and `av1/common/av1_loopfilter.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design the 8- and high-bit-depth edge layout and 4-, 6-, 8-, and 14-tap kernels for SIMD-first traversal, with sharpness thresholds, still-frame intra filter-level derivation, transform-edge selection, and plane traversal shared by the exact scalar fallback. Implement and verify both through the same contract before enabling the stage. This is normative AV1 image reconstruction and adds neither generic ISO BMFF models nor retained video reference state. | -| `Av1CdefDecoder`, `Av1CdefKernels`, and CDEF-unit strength storage | AV1 sections 7.15.2 through 7.15.4 constrained directional enhancement filtering | libaom `av1/common/cdef.c`, `av1/common/cdef_block.c`, `av1/common/cdef.h`, and `av1/common/cdef_block.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design direction search and constrained primary/secondary filtering around vector-friendly block storage and SIMD-first kernels, with variance adjustment, subsampling direction conversion, skipped-8x8 selection, and frame-edge sentinel behavior shared by the scalar fallback. Use a frame-owned source snapshot so filtering never consumes already modified samples. This is normative AV1 still-image reconstruction and introduces no ISO BMFF, track, timing, or sequence-playback surface. | +| `Av1CdefDecoder`, `Av1CdefFilter`, and CDEF-unit strength storage | AV1 sections 7.15.2 through 7.15.4 constrained directional enhancement filtering | libaom `av1/common/cdef.c`, `av1/common/cdef_block.c`, `av1/common/cdef.h`, and `av1/common/cdef_block.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design direction search and constrained primary/secondary filtering around vector-friendly block storage and SIMD-first kernels, with variance adjustment, subsampling direction conversion, skipped-8x8 selection, and frame-edge sentinel behavior shared by the scalar fallback. Use a frame-owned source snapshot so filtering never consumes already modified samples. This is normative AV1 still-image reconstruction and introduces no ISO BMFF, track, timing, or sequence-playback surface. | | `Av1SuperResolutionDecoder`, `Av1SuperResolutionKernels`, frame-size derivation, and decoded-image dimensions | AV1 section 7.16 normative super-resolution upscaling | libaom `av1/common/resize.c`, `av1/common/resize.h`, `av1/common/convolve.c`, and `aom_dsp/aom_filter.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Port the fixed 64-phase, 8-tap horizontal filter, phase/step derivation, replicated frame edges, chroma width rounding, signed rounding, and 8/10/12-bit clipping. Reuse ImageSharp's existing cross-platform `Vector128_.MultiplyAddAdjacent` helper for the exact eight-coefficient dot product with a scalar fallback. Generic image resizing is not normative AV1 super-resolution. This adds no track, timing, fragment, animation, or generic ISO BMFF model. | | `Av1TileReader` loop-restoration unit syntax, `Av1SymbolDecoder` restoration distributions/subexponential codes, and `Av1FrameInfo` unit storage | AV1 section 5.11.57 `read_lr` and `read_lr_unit` syntax | libaom `av1/decoder/decodeframe.c`, `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/entropymode.c`, `aom_dsp/binary_codes_reader.c`, and `aom_dsp/recenter.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Decode tile-local switchable/Wiener/self-guided selections, finite reference-subexponential coefficients, chroma Wiener windows, self-guided parameter sets, super-resolution-adjusted unit corners, and the AV1 nearest-unit-count rule into frame-owned per-plane grids. This is compressed still-image syntax and adds no movie, track, timing, fragment, audio, or sequence surface. | | `Av1WienerFilter` | AV1 sections 7.17.4 and 7.17.5 Wiener restoration filtering and coefficient derivation | libaom `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/convolve.c`, and `av1/common/convolve.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the implicit center-sample contribution, separable horizontal/vertical rounding, bit-depth-dependent 16-bit intermediate range, and final 8/10/12-bit clipping. Reuse `Vector128_.MultiplyAddAdjacent` for the contiguous horizontal eight-tap product with an exact scalar fallback. Keep the restoration stage disabled until stripe boundaries and self-guided filtering are both complete. | @@ -351,7 +351,7 @@ This assessment was reconciled with the source tree on 2026-08-26. Unless a resu - The single-still `Av1Decoder` path now parses tile state before allocating and reconstructing one independently decodable frame, and it disposes the reconstruction planes after pixel conversion. It deliberately does not retain animation/video reference frames or implement `show_existing_frame` playback state. - Transform coefficient entropy derivation and updates now address the above contexts relative to the tile column and the left contexts relative to the current superblock row, preserve luma coordinates independently of chroma subsampling, and test every packed context entry for the libaom any-nonzero rule. Extended vertical partition updates advance the mode-information column rather than the row. The existing multi-superblock 4:4:4 AVIF fixture now completes tile parsing; independent coefficient-context vectors across tile boundaries, chroma layouts, bit depths, and edge-clipped transforms remain required. -- The reconstruction pipeline now records plane-relative transform geometry, preserves tile-local delta-Q and delta-LF predictors, derives segmentation and reference-adjusted filter levels, and runs the exact AV1 4-, 6-, 8-, and 14-tap deblocking kernels in normative vertical-then-horizontal order. Deblocking uses the same closed edge-operator architecture as the HEVC filter, with operators specialized by sample storage and orientation, `Vector128` lanes representing the four rows or columns along an edge, and an allocation-free scalar fallback for disabled intrinsics. Exact native-plane comparison with pinned scalar libaom output now verifies active deblocking and complete reconstruction for real 8-bit 4:2:0, 10-bit 4:4:4, and 12-bit 4:4:4 content; genuine AVIF containers separately verify presentation and public bit-depth metadata. The pipeline then applies scalar CDEF direction search, luma variance adjustment, primary and secondary constrained taps, chroma direction conversion, high-bit-depth scaling, skipped-block selection, and frame-edge sentinel handling from immutable per-plane snapshots. Active super-resolution derives the Appendix A bounded coded width and applies the exact 64-phase, 8-tap horizontal filter with aligned reconstruction-edge input, 8/10/12-bit clipping, and the existing cross-platform `Vector128_.MultiplyAddAdjacent` helper. Loop restoration follows super-resolution, preserves the required pre-CDEF deblocked context at internal stripes, and applies decoded Wiener or self-guided units from immutable plane snapshots. The visible still-image path then applies the complete self-contained film-grain parameter set after all in-loop filters. Independent 8-, 10-, and 12-bit vectors exercising CDEF, super-resolution, restoration, and grain remain required before those stages have external pixel-level verification. +- The reconstruction pipeline now records plane-relative transform geometry, preserves tile-local delta-Q and delta-LF predictors, derives segmentation and reference-adjusted filter levels, and runs the exact AV1 4-, 6-, 8-, and 14-tap deblocking kernels in normative vertical-then-horizontal order. Deblocking uses the same closed edge-operator architecture as the HEVC filter, with operators specialized by sample storage and orientation, `Vector128` lanes representing the four rows or columns along an edge, and an allocation-free scalar fallback for disabled intrinsics. Exact native-plane comparison with pinned scalar libaom output now verifies active deblocking and complete reconstruction for real 8-bit 4:2:0, 10-bit 4:4:4, and 12-bit 4:4:4 content; genuine AVIF containers separately verify presentation and public bit-depth metadata. The pipeline then applies CDEF through one semantic filter architecture: paired AVX2 and single-block `Vector128` direction analysis, closed primary/secondary strength operators, packed 4x4/4x8/8x4/8x8 constrained filtering, byte/16-bit output operators, and an exact allocation-free scalar fallback. The decoder still uses heap-allocated immutable plane snapshots and direction/variance maps; moving those to allocator-owned memory and using paired direction traversal is the next CDEF step. Active super-resolution derives the Appendix A bounded coded width and applies the exact 64-phase, 8-tap horizontal filter with aligned reconstruction-edge input, 8/10/12-bit clipping, and the existing cross-platform `Vector128_.MultiplyAddAdjacent` helper. Loop restoration follows super-resolution, preserves the required pre-CDEF deblocked context at internal stripes, and applies decoded Wiener or self-guided units from immutable plane snapshots. The visible still-image path then applies the complete self-contained film-grain parameter set after all in-loop filters. Independent 8-, 10-, and 12-bit vectors exercising CDEF, super-resolution, restoration, and grain remain required before those stages have external pixel-level verification. - Palette mode now reads the normative luma/chroma mode and size CDFs, neighbor color caches, high-bit-depth color syntax, diagonal color-index maps, and clipped-edge padding. Reconstruction retains libaom's exact indexed-palette contract while using repeated eight-entry tables and native lane-local shuffles across runtime-selected `Vector512`, `Vector256`, and `Vector128` paths before an exact scalar fallback. `FeatureTestRunner` verifies palette sizes 2-8, transform widths 4-64, padded rows, and 8/10/12-bit samples. This path is not production-complete until its syntax and reconstruction are verified with an independently encoded palette AVIF fixture. - Non-reduced still-image sequence parsing now consumes decoder-model operating parameters, temporal presentation fields, and OBU-layer-applicable buffer-removal fields only to preserve AV1 bit alignment. The scheduling values are not retained, and no movie, track, timing, playback, or generic ISO BMFF surface has been introduced. Existing focused sequence-header coverage exercises only reduced-still syntax, so an independent non-reduced still AVIF vector remains required. - Loop-restoration unit parsing records tile-local switchable/Wiener/self-guided filter selections and coefficients in frame-owned plane grids, including super-resolution-adjusted unit corners and the corrected conditional 64x64-superblock unit-size bit. The active restoration stage implements the normative unit geometry, striped deblocked boundaries, Wiener filtering, self-guided projection, and 8/10/12-bit clipping, while reusing the existing SIMD-backed super-resolution and adjacent multiply/add primitives. Independently encoded fixtures covering every parameter set, plane layout, bit depth, and frame-edge geometry are still required. Other normative independently decodable still-image syntax paths still contain `NotImplementedException` or equivalent unsupported branches. Tile-local palette CDF adaptation is present; the remaining still-image frame-context behavior requires a separate source audit without introducing sequence playback state. @@ -505,6 +505,9 @@ Implement and verify in dependency order: - [x] Implement allocation-free SIMD-first 4-, 6-, 8-, and 14-tap filtering for vertical and horizontal edges in 8/10/12-bit storage through closed edge operators, with exact scalar fallback and `FeatureTestRunner` parity against an independent definition. - [x] Verify deblocking syntax, filter-level derivation, and boundary traversal with independently encoded 8/10/12-bit AV1 samples and exact scalar-libaom planes; verify presented reconstruction and public precision with genuine AVIF containers at every supported bit depth. - [ ] CDEF. + - [x] Replace the scalar helper bucket with an allocation-free SIMD-first `Av1CdefFilter`: vector direction analysis, 4x4/4x8/8x4/8x8 constrained filtering, byte/16-bit output operators, and one exact scalar fallback verified through `FeatureTestRunner`. + - [ ] Move the immutable deblocked-plane snapshot and frame-wide direction/variance maps to allocator-owned decoder memory, preserving luma-first direction derivation, skipped-block selection, asymmetric chroma direction conversion, and frame-edge sentinels. + - [ ] Verify CDEF syntax, strength derivation, unit/block traversal, subsampling, frame edges, and presented reconstruction with independently encoded active-CDEF AV1/AVIF fixtures across 8/10/12-bit samples. - [ ] Super-resolution scaling. - [ ] Loop restoration. - [ ] Frame padding and film-grain synthesis when signaled. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefDecoder.cs index a178d3d81..7455792f9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefDecoder.cs @@ -130,7 +130,7 @@ internal class Av1CdefDecoder // CDEF output must never become input to a later block. The sentinel border also makes // frame-edge taps follow AV1 without exposing the frame buffer's prediction padding. ushort[] source = new ushort[(planeHeight + (SourceBorder * 2)) * sourceStride]; - Array.Fill(source, Av1CdefKernels.VeryLarge); + Array.Fill(source, Av1CdefFilter.VeryLarge); Span lowBitDepthDestination = default; Span highBitDepthDestination = default; @@ -236,7 +236,7 @@ internal class Av1CdefDecoder if (plane == Av1Plane.Y) { - directions[directionIndex] = Av1CdefKernels.FindDirection( + directions[directionIndex] = Av1CdefFilter.FindDirection( source, sourceOffset, sourceStride, @@ -250,7 +250,7 @@ internal class Av1CdefDecoder } int filteredPrimaryStrength = plane == Av1Plane.Y - ? Av1CdefKernels.AdjustStrength(primaryStrength, variances[directionIndex]) + ? Av1CdefFilter.AdjustStrength(primaryStrength, variances[directionIndex]) : primaryStrength; if (filteredPrimaryStrength == 0 && secondaryStrength == 0) @@ -261,10 +261,10 @@ internal class Av1CdefDecoder // Secondary-only filtering uses direction zero; otherwise chroma remaps the // luma direction into its asymmetrically subsampled sample grid when required. int direction = primaryStrength != 0 - ? Av1CdefKernels.ConvertDirection(directions[directionIndex], subsamplingX, subsamplingY) + ? Av1CdefFilter.ConvertDirection(directions[directionIndex], subsamplingX, subsamplingY) : 0; - Av1CdefKernels.FilterBlock( + Av1CdefFilter.FilterBlock( source, sourceOffset, sourceStride, diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs new file mode 100644 index 000000000..2f93844d2 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs @@ -0,0 +1,1703 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; + +/// +/// Applies AV1 constrained directional enhancement filtering and derives the dominant direction of reconstructed blocks. +/// +internal static class Av1CdefFilter +{ + /// + /// The sample value used in the bordered source plane for neighbors outside the coded frame. + /// + public const ushort VeryLarge = 0x4000; + + /// + /// Defines storage-specific writes for one filtered row. + /// + /// The destination sample storage type. + private interface IOutputOperator + where TSample : unmanaged + { + /// + /// Stores four or eight filtered samples from the low vector lanes. + /// + /// The first element in the destination plane. + /// The offset of the first sample to write. + /// The filtered samples in the low lanes. + /// The number of valid lanes. + public static abstract void StoreVector(ref TSample destination, int offset, Vector128 value, int count); + + /// + /// Stores one filtered sample. + /// + /// The first element in the destination plane. + /// The offset of the sample to write. + /// The filtered sample. + public static abstract void StoreScalar(ref TSample destination, int offset, int value); + } + + /// + /// Defines which groups of directional taps participate in one closed filter kernel. + /// + private interface IFilterOperator + { + /// + /// Gets a value indicating whether the primary directional taps are enabled. + /// + public static abstract bool EnablePrimary { get; } + + /// + /// Gets a value indicating whether the secondary off-axis taps are enabled. + /// + public static abstract bool EnableSecondary { get; } + } + + /// + /// Finds the dominant direction of an 8x8 luma block and its directional variance. + /// + /// The bordered, deblocked source plane. + /// The offset of the block's top-left sample. + /// The number of samples between adjacent source rows. + /// The number of bits above the eight-bit analysis precision. + /// Receives the variance difference between the selected and orthogonal directions. + /// The zero-based AV1 direction index. + public static int FindDirection( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + int coefficientShift, + out int variance) + { + if (Vector128.IsHardwareAccelerated) + { + return FindDirectionVector(source, sourceOffset, sourceStride, coefficientShift, out variance); + } + + return FindDirectionScalar(source, sourceOffset, sourceStride, coefficientShift, out variance); + } + + /// + /// Finds the dominant directions and directional variances of two independent 8x8 luma blocks. + /// + /// The bordered, deblocked source plane containing both blocks. + /// The offset of the first block's top-left sample. + /// The offset of the second block's top-left sample. + /// The number of samples between adjacent source rows. + /// The number of bits above the eight-bit analysis precision. + /// Receives the first block's zero-based AV1 direction index. + /// Receives the first block's directional variance. + /// Receives the second block's zero-based AV1 direction index. + /// Receives the second block's directional variance. + public static void FindDirections( + ReadOnlySpan source, + int firstSourceOffset, + int secondSourceOffset, + int sourceStride, + int coefficientShift, + out int firstDirection, + out int firstVariance, + out int secondDirection, + out int secondVariance) + { + if (Avx2.IsSupported) + { + FindDirectionsVector( + source, + firstSourceOffset, + secondSourceOffset, + sourceStride, + coefficientShift, + out firstDirection, + out firstVariance, + out secondDirection, + out secondVariance); + + return; + } + + firstDirection = FindDirection(source, firstSourceOffset, sourceStride, coefficientShift, out firstVariance); + secondDirection = FindDirection(source, secondSourceOffset, sourceStride, coefficientShift, out secondVariance); + } + + /// + /// Adjusts a luma primary strength according to the directional variance of its 8x8 block. + /// + /// The bit-depth-scaled primary strength. + /// The directional variance returned by . + /// The variance-adjusted primary strength. + public static int AdjustStrength(int strength, int variance) + { + int varianceClass = variance >> 6; + int adjustment = varianceClass != 0 ? Math.Min(Av1Math.MostSignificantBit((uint)varianceClass), 12) : 0; + return variance != 0 ? ((strength * (4 + adjustment)) + 8) >> 4 : 0; + } + + /// + /// Converts a luma direction to the matching chroma direction for asymmetric subsampling. + /// + /// The zero-based luma direction index. + /// The horizontal chroma subsampling shift. + /// The vertical chroma subsampling shift. + /// The direction index in the chroma sample grid. + public static int ConvertDirection(int direction, int subsamplingX, int subsamplingY) + { + if (subsamplingX == subsamplingY) + { + return direction; + } + + return subsamplingX != 0 + ? direction switch + { + 0 => 7, + 1 => 0, + 2 => 2, + 3 => 4, + 4 => 5, + _ => 6 + } + : direction switch + { + 0 => 1, + 1 or 2 or 3 => 2, + 4 => 3, + 5 => 4, + 6 => 6, + _ => 0 + }; + } + + /// + /// Filters one luma or chroma block into eight-bit sample storage. + /// + /// The bordered, deblocked source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The unbordered filtered destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + public static void FilterBlock( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + Span destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + => FilterBlock( + source, + sourceOffset, + sourceStride, + destination, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + + /// + /// Filters one luma or chroma block into 16-bit sample storage. + /// + /// The bordered, deblocked source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The unbordered filtered destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + public static void FilterBlock( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + Span destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + => FilterBlock( + source, + sourceOffset, + sourceStride, + destination, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + + /// + /// Selects the packed CDEF kernel or its scalar fallback through one closed output operator. + /// + /// The destination sample storage type. + /// The storage-specific output operator. + /// The bordered, deblocked source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The unbordered filtered destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + private static void FilterBlock( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + Span destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + where TSample : unmanaged + where TOutputOperator : struct, IOutputOperator + { + ref ushort sourceBase = ref MemoryMarshal.GetReference(source); + ref TSample destinationBase = ref MemoryMarshal.GetReference(destination); + + // libaom selects one of four closed kernels from the two strength flags. The semantic operator makes the same + // choice once per block so the JIT removes primary/secondary mode branches from every row and tap. + if (primaryStrength != 0) + { + if (secondaryStrength != 0) + { + FilterBlock( + ref sourceBase, + sourceOffset, + sourceStride, + ref destinationBase, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + } + else + { + FilterBlock( + ref sourceBase, + sourceOffset, + sourceStride, + ref destinationBase, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + } + + return; + } + + if (secondaryStrength != 0) + { + FilterBlock( + ref sourceBase, + sourceOffset, + sourceStride, + ref destinationBase, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + + return; + } + + FilterBlock( + ref sourceBase, + sourceOffset, + sourceStride, + ref destinationBase, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + } + + /// + /// Selects the packed or scalar implementation of one closed filter kernel. + /// + /// The destination sample storage type. + /// The storage-specific output operator. + /// The enabled directional-tap operator. + /// The first element in the bordered source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The first element in the destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + private static void FilterBlock( + ref ushort source, + int sourceOffset, + int sourceStride, + ref TSample destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + where TSample : unmanaged + where TOutputOperator : struct, IOutputOperator + where TFilterOperator : struct, IFilterOperator + { + if (Avx2.IsSupported) + { + FilterBlockWideVector( + ref source, + sourceOffset, + sourceStride, + ref destination, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + FilterBlockVector( + ref source, + sourceOffset, + sourceStride, + ref destination, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + + return; + } + + FilterBlockScalar( + ref source, + sourceOffset, + sourceStride, + ref destination, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + primaryDamping, + secondaryDamping, + coefficientShift, + blockWidth, + blockHeight); + } + + /// + /// Applies one packed CDEF kernel to two 8-wide rows or four 4-wide rows at a time. + /// + /// The destination sample storage type. + /// The storage-specific output operator. + /// The enabled directional-tap operator. + /// The first element in the bordered source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The first element in the destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + private static void FilterBlockWideVector( + ref ushort source, + int sourceOffset, + int sourceStride, + ref TSample destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + where TSample : unmanaged + where TOutputOperator : struct, IOutputOperator + where TFilterOperator : struct, IFilterOperator + { + bool clippingRequired = TFilterOperator.EnablePrimary && TFilterOperator.EnableSecondary; + int primaryDampingShift = TFilterOperator.EnablePrimary ? Math.Max(0, primaryDamping - Av1Math.MostSignificantBit((uint)primaryStrength)) : 0; + int secondaryDampingShift = TFilterOperator.EnableSecondary ? Math.Max(0, secondaryDamping - Av1Math.MostSignificantBit((uint)secondaryStrength)) : 0; + int primaryTapSet = (primaryStrength >> coefficientShift) & 1; + int primaryNearOffset = TFilterOperator.EnablePrimary ? GetDirectionOffset(direction, 0, sourceStride) : 0; + int primaryFarOffset = TFilterOperator.EnablePrimary ? GetDirectionOffset(direction, 1, sourceStride) : 0; + int secondaryNearOffset0 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 2) & 7, 0, sourceStride) : 0; + int secondaryFarOffset0 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 2) & 7, 1, sourceStride) : 0; + int secondaryNearOffset1 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 6) & 7, 0, sourceStride) : 0; + int secondaryFarOffset1 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 6) & 7, 1, sourceStride) : 0; + Vector256 primaryNearWeight = Vector256.Create((short)(primaryTapSet == 0 ? 4 : 3)); + Vector256 primaryFarWeight = Vector256.Create((short)(primaryTapSet == 0 ? 2 : 3)); + Vector256 secondaryNearWeight = Vector256.Create((short)2); + Vector256 secondaryFarWeight = Vector256.Create((short)1); + Vector256 sentinel = Vector256.Create((short)VeryLarge); + Vector256 zero = Vector256.Zero; + Vector256 rounding = Vector256.Create((short)8); + Vector256 one = Vector256.Create((short)1); + int rowsPerBatch = blockWidth == 8 ? 2 : 4; + + // The 256-bit lane layout follows libaom: two complete 8-wide rows, or four complete 4-wide rows. Directional + // offsets therefore remain ordinary source offsets, while all constrain, weight, clip, and round operations + // advance several output rows together without crossing a row boundary inside any 128-bit lane. + for (int row = 0; row < blockHeight; row += rowsPerBatch) + { + int sourceIndex = sourceOffset + (row * sourceStride); + Vector256 sample = LoadRows(ref source, sourceIndex, sourceStride, blockWidth); + Vector256 sum = zero; + Vector256 minimum = sample; + Vector256 maximum = sample; + + for (int tap = 0; tap < 2; tap++) + { + if (TFilterOperator.EnablePrimary) + { + int offset = tap == 0 ? primaryNearOffset : primaryFarOffset; + Vector256 neighbor0 = LoadRows(ref source, sourceIndex + offset, sourceStride, blockWidth); + Vector256 neighbor1 = LoadRows(ref source, sourceIndex - offset, sourceStride, blockWidth); + Vector256 constrained = Constrain(neighbor0, sample, primaryStrength, primaryDampingShift) + + Constrain(neighbor1, sample, primaryStrength, primaryDampingShift); + + sum += constrained * (tap == 0 ? primaryNearWeight : primaryFarWeight); + if (clippingRequired) + { + minimum = Vector256.Min(minimum, Vector256.Min(neighbor0, neighbor1)); + maximum = MaximumIgnoringSentinel(maximum, neighbor0, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor1, sentinel); + } + } + + if (TFilterOperator.EnableSecondary) + { + int offset0 = tap == 0 ? secondaryNearOffset0 : secondaryFarOffset0; + int offset1 = tap == 0 ? secondaryNearOffset1 : secondaryFarOffset1; + Vector256 neighbor0 = LoadRows(ref source, sourceIndex + offset0, sourceStride, blockWidth); + Vector256 neighbor1 = LoadRows(ref source, sourceIndex - offset0, sourceStride, blockWidth); + Vector256 neighbor2 = LoadRows(ref source, sourceIndex + offset1, sourceStride, blockWidth); + Vector256 neighbor3 = LoadRows(ref source, sourceIndex - offset1, sourceStride, blockWidth); + Vector256 constrained = Constrain(neighbor0, sample, secondaryStrength, secondaryDampingShift) + + Constrain(neighbor1, sample, secondaryStrength, secondaryDampingShift) + + Constrain(neighbor2, sample, secondaryStrength, secondaryDampingShift) + + Constrain(neighbor3, sample, secondaryStrength, secondaryDampingShift); + + sum += constrained * (tap == 0 ? secondaryNearWeight : secondaryFarWeight); + if (clippingRequired) + { + minimum = Vector256.Min(minimum, Vector256.Min(Vector256.Min(neighbor0, neighbor1), Vector256.Min(neighbor2, neighbor3))); + maximum = MaximumIgnoringSentinel(maximum, neighbor0, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor1, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor2, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor3, sentinel); + } + } + } + + Vector256 correction = (sum >> 15) & one; + Vector256 filtered = sample + ((sum + rounding - correction) >> 4); + if (clippingRequired) + { + filtered = Vector256.Min(Vector256.Max(filtered, minimum), maximum); + } + + StoreRows( + ref destination, + destinationOffset + (row * destinationStride), + destinationStride, + filtered, + blockWidth); + } + } + + /// + /// Finds the dominant direction with eight signed 16-bit lanes representing one complete source row. + /// + /// The bordered, deblocked source plane. + /// The offset of the block's top-left sample. + /// The number of samples between adjacent source rows. + /// The number of bits above the eight-bit analysis precision. + /// Receives the variance difference between the selected and orthogonal directions. + /// The zero-based AV1 direction index. + private static int FindDirectionVector( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + int coefficientShift, + out int variance) + { + ref ushort sourceBase = ref MemoryMarshal.GetReference(source); + InlineArray8> lines = default; + Vector128 analysisBias = Vector128.Create((short)128); + + for (int row = 0; row < 8; row++) + { + Vector128 samples = Vector128.LoadUnsafe(ref sourceBase, (nuint)(sourceOffset + (row * sourceStride))); + + // AV1 direction selection is defined in an eight-bit domain. The signed bias keeps every accumulated line sum + // within Int16 while preserving identical direction and variance results for 8-, 10-, and 12-bit samples. + lines[row] = (samples >> coefficientShift).AsInt16() - analysisBias; + } + + // Vector lanes are written in memory order. These are the low-to-high forms of libaom's set-style constants. + Vector128 foldWeights0 = Vector128.Create(840, 420, 280, 210); + Vector128 foldWeights1 = Vector128.Create(168, 140, 120, 105); + Vector128 diagonalWeights0 = Vector128.Create(0, 0, 420, 210); + Vector128 diagonalWeights1 = Vector128.Create(140, 105, 105, 105); + InlineArray8 costs = default; + ref int costBase = ref costs[0]; + + // The first pass evaluates directions 4..7. Rotating the block counter-clockwise lets the identical arithmetic + // evaluate directions 0..3, exactly matching libaom's portable vector implementation. + ComputeDirectionCosts(ref lines, foldWeights0, foldWeights1, diagonalWeights0, diagonalWeights1).StoreUnsafe(ref costBase, 4); + ReverseTranspose(ref lines); + ComputeDirectionCosts(ref lines, foldWeights0, foldWeights1, diagonalWeights0, diagonalWeights1).StoreUnsafe(ref costBase); + + return SelectDirection(ref costs, out variance); + } + + /// + /// Finds two dominant directions with each 128-bit lane of a 256-bit vector representing one independent block. + /// + /// The bordered, deblocked source plane containing both blocks. + /// The offset of the first block's top-left sample. + /// The offset of the second block's top-left sample. + /// The number of samples between adjacent source rows. + /// The number of bits above the eight-bit analysis precision. + /// Receives the first block's zero-based AV1 direction index. + /// Receives the first block's directional variance. + /// Receives the second block's zero-based AV1 direction index. + /// Receives the second block's directional variance. + private static void FindDirectionsVector( + ReadOnlySpan source, + int firstSourceOffset, + int secondSourceOffset, + int sourceStride, + int coefficientShift, + out int firstDirection, + out int firstVariance, + out int secondDirection, + out int secondVariance) + { + ref ushort sourceBase = ref MemoryMarshal.GetReference(source); + InlineArray8> lines = default; + Vector256 analysisBias = Vector256.Create((short)128); + + for (int row = 0; row < 8; row++) + { + Vector128 first = Vector128.LoadUnsafe(ref sourceBase, (nuint)(firstSourceOffset + (row * sourceStride))); + Vector128 second = Vector128.LoadUnsafe(ref sourceBase, (nuint)(secondSourceOffset + (row * sourceStride))); + + // Keeping one block in each 128-bit lane is the critical libaom layout: every byte shift, unpack, multiply, + // and transpose remains lane-local while one AVX2 instruction advances both direction searches. + lines[row] = (Vector256.Create(first, second) >> coefficientShift).AsInt16() - analysisBias; + } + + Vector128 foldWeights0 = Vector128.Create(840, 420, 280, 210); + Vector128 foldWeights1 = Vector128.Create(168, 140, 120, 105); + Vector128 diagonalWeights0 = Vector128.Create(0, 0, 420, 210); + Vector128 diagonalWeights1 = Vector128.Create(140, 105, 105, 105); + Vector256 packedFoldWeights0 = Vector256.Create(foldWeights0, foldWeights0); + Vector256 packedFoldWeights1 = Vector256.Create(foldWeights1, foldWeights1); + Vector256 packedDiagonalWeights0 = Vector256.Create(diagonalWeights0, diagonalWeights0); + Vector256 packedDiagonalWeights1 = Vector256.Create(diagonalWeights1, diagonalWeights1); + InlineArray8 firstCosts = default; + InlineArray8 secondCosts = default; + ref int firstCostBase = ref firstCosts[0]; + ref int secondCostBase = ref secondCosts[0]; + + Vector256 direction47 = ComputeDirectionCosts( + ref lines, + packedFoldWeights0, + packedFoldWeights1, + packedDiagonalWeights0, + packedDiagonalWeights1); + + direction47.GetLower().StoreUnsafe(ref firstCostBase, 4); + direction47.GetUpper().StoreUnsafe(ref secondCostBase, 4); + ReverseTranspose(ref lines); + Vector256 direction03 = ComputeDirectionCosts( + ref lines, + packedFoldWeights0, + packedFoldWeights1, + packedDiagonalWeights0, + packedDiagonalWeights1); + + direction03.GetLower().StoreUnsafe(ref firstCostBase); + direction03.GetUpper().StoreUnsafe(ref secondCostBase); + firstDirection = SelectDirection(ref firstCosts, out firstVariance); + secondDirection = SelectDirection(ref secondCosts, out secondVariance); + } + + /// + /// Computes four adjacent AV1 direction costs from eight packed source rows. + /// + /// The eight signed, biased source rows. + /// The line-length weights for the first four folded pairs. + /// The line-length weights for the second four folded pairs. + /// The first line-length weights for the shallow diagonal directions. + /// The second line-length weights for the shallow diagonal directions. + /// The four costs ordered by increasing direction within the current orientation. + private static Vector128 ComputeDirectionCosts( + ref InlineArray8> lines, + Vector128 foldWeights0, + Vector128 foldWeights1, + Vector128 diagonalWeights0, + Vector128 diagonalWeights1) + { + // Byte-lane shifts move whole Int16 samples while inserting zeroes. Each shifted row therefore lands in the + // vector lanes for one geometric line without gathers or a per-block partial-sum buffer. + Vector128 partial4A = Vector128_.ShiftLeftBytesInVector(lines[0].AsByte(), 14).AsInt16(); + Vector128 partial4B = Vector128_.ShiftRightBytesInVector(lines[0].AsByte(), 2).AsInt16(); + partial4A += Vector128_.ShiftLeftBytesInVector(lines[1].AsByte(), 12).AsInt16(); + partial4B += Vector128_.ShiftRightBytesInVector(lines[1].AsByte(), 4).AsInt16(); + Vector128 pair = lines[0] + lines[1]; + Vector128 partial5A = Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 10).AsInt16(); + Vector128 partial5B = Vector128_.ShiftRightBytesInVector(pair.AsByte(), 6).AsInt16(); + Vector128 partial7A = Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 4).AsInt16(); + Vector128 partial7B = Vector128_.ShiftRightBytesInVector(pair.AsByte(), 12).AsInt16(); + Vector128 partial6 = pair; + + partial4A += Vector128_.ShiftLeftBytesInVector(lines[2].AsByte(), 10).AsInt16(); + partial4B += Vector128_.ShiftRightBytesInVector(lines[2].AsByte(), 6).AsInt16(); + partial4A += Vector128_.ShiftLeftBytesInVector(lines[3].AsByte(), 8).AsInt16(); + partial4B += Vector128_.ShiftRightBytesInVector(lines[3].AsByte(), 8).AsInt16(); + pair = lines[2] + lines[3]; + partial5A += Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 8).AsInt16(); + partial5B += Vector128_.ShiftRightBytesInVector(pair.AsByte(), 8).AsInt16(); + partial7A += Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 6).AsInt16(); + partial7B += Vector128_.ShiftRightBytesInVector(pair.AsByte(), 10).AsInt16(); + partial6 += pair; + + partial4A += Vector128_.ShiftLeftBytesInVector(lines[4].AsByte(), 6).AsInt16(); + partial4B += Vector128_.ShiftRightBytesInVector(lines[4].AsByte(), 10).AsInt16(); + partial4A += Vector128_.ShiftLeftBytesInVector(lines[5].AsByte(), 4).AsInt16(); + partial4B += Vector128_.ShiftRightBytesInVector(lines[5].AsByte(), 12).AsInt16(); + pair = lines[4] + lines[5]; + partial5A += Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 6).AsInt16(); + partial5B += Vector128_.ShiftRightBytesInVector(pair.AsByte(), 10).AsInt16(); + partial7A += Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 8).AsInt16(); + partial7B += Vector128_.ShiftRightBytesInVector(pair.AsByte(), 8).AsInt16(); + partial6 += pair; + + partial4A += Vector128_.ShiftLeftBytesInVector(lines[6].AsByte(), 2).AsInt16(); + partial4B += Vector128_.ShiftRightBytesInVector(lines[6].AsByte(), 14).AsInt16(); + partial4A += lines[7]; + pair = lines[6] + lines[7]; + partial5A += Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 4).AsInt16(); + partial5B += Vector128_.ShiftRightBytesInVector(pair.AsByte(), 12).AsInt16(); + partial7A += Vector128_.ShiftLeftBytesInVector(pair.AsByte(), 10).AsInt16(); + partial7B += Vector128_.ShiftRightBytesInVector(pair.AsByte(), 6).AsInt16(); + partial6 += pair; + + Vector128 partial4Cost = FoldDirectionPartials(partial4A, partial4B, foldWeights0, foldWeights1); + Vector128 partial5Cost = FoldDirectionPartials(partial5A, partial5B, diagonalWeights0, diagonalWeights1); + Vector128 partial7Cost = FoldDirectionPartials(partial7A, partial7B, diagonalWeights0, diagonalWeights1); + Vector128 partial6Cost = Vector128_.MultiplyAddAdjacent(partial6, partial6) * Vector128.Create(105); + return HorizontalSumFour(partial4Cost, partial5Cost, partial6Cost, partial7Cost); + } + + /// + /// Computes four adjacent AV1 direction costs for two blocks packed into independent 128-bit lanes. + /// + /// The eight signed, biased source rows for both blocks. + /// The line-length weights for the first four folded pairs. + /// The line-length weights for the second four folded pairs. + /// The first line-length weights for the shallow diagonal directions. + /// The second line-length weights for the shallow diagonal directions. + /// The four costs per block ordered by increasing direction within the current orientation. + private static Vector256 ComputeDirectionCosts( + ref InlineArray8> lines, + Vector256 foldWeights0, + Vector256 foldWeights1, + Vector256 diagonalWeights0, + Vector256 diagonalWeights1) + { + Vector256 partial4A = Avx2.ShiftLeftLogical128BitLane(lines[0].AsByte(), 14).AsInt16(); + Vector256 partial4B = Avx2.ShiftRightLogical128BitLane(lines[0].AsByte(), 2).AsInt16(); + partial4A += Avx2.ShiftLeftLogical128BitLane(lines[1].AsByte(), 12).AsInt16(); + partial4B += Avx2.ShiftRightLogical128BitLane(lines[1].AsByte(), 4).AsInt16(); + Vector256 pair = lines[0] + lines[1]; + Vector256 partial5A = Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 10).AsInt16(); + Vector256 partial5B = Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 6).AsInt16(); + Vector256 partial7A = Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 4).AsInt16(); + Vector256 partial7B = Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 12).AsInt16(); + Vector256 partial6 = pair; + + partial4A += Avx2.ShiftLeftLogical128BitLane(lines[2].AsByte(), 10).AsInt16(); + partial4B += Avx2.ShiftRightLogical128BitLane(lines[2].AsByte(), 6).AsInt16(); + partial4A += Avx2.ShiftLeftLogical128BitLane(lines[3].AsByte(), 8).AsInt16(); + partial4B += Avx2.ShiftRightLogical128BitLane(lines[3].AsByte(), 8).AsInt16(); + pair = lines[2] + lines[3]; + partial5A += Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 8).AsInt16(); + partial5B += Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 8).AsInt16(); + partial7A += Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 6).AsInt16(); + partial7B += Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 10).AsInt16(); + partial6 += pair; + + partial4A += Avx2.ShiftLeftLogical128BitLane(lines[4].AsByte(), 6).AsInt16(); + partial4B += Avx2.ShiftRightLogical128BitLane(lines[4].AsByte(), 10).AsInt16(); + partial4A += Avx2.ShiftLeftLogical128BitLane(lines[5].AsByte(), 4).AsInt16(); + partial4B += Avx2.ShiftRightLogical128BitLane(lines[5].AsByte(), 12).AsInt16(); + pair = lines[4] + lines[5]; + partial5A += Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 6).AsInt16(); + partial5B += Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 10).AsInt16(); + partial7A += Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 8).AsInt16(); + partial7B += Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 8).AsInt16(); + partial6 += pair; + + partial4A += Avx2.ShiftLeftLogical128BitLane(lines[6].AsByte(), 2).AsInt16(); + partial4B += Avx2.ShiftRightLogical128BitLane(lines[6].AsByte(), 14).AsInt16(); + partial4A += lines[7]; + pair = lines[6] + lines[7]; + partial5A += Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 4).AsInt16(); + partial5B += Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 12).AsInt16(); + partial7A += Avx2.ShiftLeftLogical128BitLane(pair.AsByte(), 10).AsInt16(); + partial7B += Avx2.ShiftRightLogical128BitLane(pair.AsByte(), 6).AsInt16(); + partial6 += pair; + + Vector256 partial4Cost = FoldDirectionPartials(partial4A, partial4B, foldWeights0, foldWeights1); + Vector256 partial5Cost = FoldDirectionPartials(partial5A, partial5B, diagonalWeights0, diagonalWeights1); + Vector256 partial7Cost = FoldDirectionPartials(partial7A, partial7B, diagonalWeights0, diagonalWeights1); + Vector256 partial6Cost = Avx2.MultiplyAddAdjacent(partial6, partial6) * Vector256.Create(105); + return HorizontalSumFour(partial4Cost, partial5Cost, partial6Cost, partial7Cost); + } + + /// + /// Squares, weights, and combines the two halves of one set of directional line sums. + /// + /// The first eight line sums. + /// The remaining seven line sums followed by zero. + /// The first four line-length weights. + /// The second four line-length weights. + /// Four packed weighted partial costs. + private static Vector128 FoldDirectionPartials( + Vector128 partialA, + Vector128 partialB, + Vector128 weights0, + Vector128 weights1) + { + // Reversal aligns equally long lines. Interleaving then gives MultiplyAddAdjacent the [x,y] pairs whose + // squared magnitudes share one line-length weight, including the unpaired centre line with an inserted zero. + partialB = Vector128.ShuffleNative(partialB, Vector128.Create((short)6, 5, 4, 3, 2, 1, 0, 7)); + Vector128 originalA = partialA; + partialA = Vector128_.UnpackLow(partialA, partialB); + partialB = Vector128_.UnpackHigh(originalA, partialB); + Vector128 lower = Vector128_.MultiplyAddAdjacent(partialA, partialA) * weights0; + Vector128 upper = Vector128_.MultiplyAddAdjacent(partialB, partialB) * weights1; + return lower + upper; + } + + /// + /// Squares, weights, and combines directional line sums for two independent packed blocks. + /// + /// The first eight line sums in each 128-bit lane. + /// The remaining seven line sums followed by zero in each 128-bit lane. + /// The first four line-length weights in each 128-bit lane. + /// The second four line-length weights in each 128-bit lane. + /// Four packed weighted partial costs per block. + private static Vector256 FoldDirectionPartials( + Vector256 partialA, + Vector256 partialB, + Vector256 weights0, + Vector256 weights1) + { + Vector128 laneShuffle = Vector128.Create((byte)12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15); + partialB = Avx2.Shuffle(partialB.AsByte(), Vector256.Create(laneShuffle, laneShuffle)).AsInt16(); + Vector256 originalA = partialA; + partialA = Avx2.UnpackLow(partialA, partialB); + partialB = Avx2.UnpackHigh(originalA, partialB); + Vector256 lower = Avx2.MultiplyAddAdjacent(partialA, partialA) * weights0; + Vector256 upper = Avx2.MultiplyAddAdjacent(partialB, partialB) * weights1; + return lower + upper; + } + + /// + /// Horizontally reduces four cost vectors into four direction costs. + /// + /// The first direction's four partial costs. + /// The second direction's four partial costs. + /// The third direction's four partial costs. + /// The fourth direction's four partial costs. + /// The four horizontally reduced costs. + private static Vector128 HorizontalSumFour( + Vector128 cost0, + Vector128 cost1, + Vector128 cost2, + Vector128 cost3) + { + Vector128 pair01Lower = Vector128_.UnpackLow(cost0, cost1); + Vector128 pair23Lower = Vector128_.UnpackLow(cost2, cost3); + Vector128 pair01Upper = Vector128_.UnpackHigh(cost0, cost1); + Vector128 pair23Upper = Vector128_.UnpackHigh(cost2, cost3); + Vector128 quad0 = Vector128_.UnpackLow(pair01Lower.AsInt64(), pair23Lower.AsInt64()).AsInt32(); + Vector128 quad1 = Vector128_.UnpackHigh(pair01Lower.AsInt64(), pair23Lower.AsInt64()).AsInt32(); + Vector128 quad2 = Vector128_.UnpackLow(pair01Upper.AsInt64(), pair23Upper.AsInt64()).AsInt32(); + Vector128 quad3 = Vector128_.UnpackHigh(pair01Upper.AsInt64(), pair23Upper.AsInt64()).AsInt32(); + return (quad0 + quad1) + (quad2 + quad3); + } + + /// + /// Horizontally reduces four cost vectors independently within both 128-bit lanes. + /// + /// The first direction's four partial costs per block. + /// The second direction's four partial costs per block. + /// The third direction's four partial costs per block. + /// The fourth direction's four partial costs per block. + /// The four horizontally reduced costs per block. + private static Vector256 HorizontalSumFour( + Vector256 cost0, + Vector256 cost1, + Vector256 cost2, + Vector256 cost3) + { + Vector256 pair01Lower = Avx2.UnpackLow(cost0, cost1); + Vector256 pair23Lower = Avx2.UnpackLow(cost2, cost3); + Vector256 pair01Upper = Avx2.UnpackHigh(cost0, cost1); + Vector256 pair23Upper = Avx2.UnpackHigh(cost2, cost3); + Vector256 quad0 = Avx2.UnpackLow(pair01Lower.AsInt64(), pair23Lower.AsInt64()).AsInt32(); + Vector256 quad1 = Avx2.UnpackHigh(pair01Lower.AsInt64(), pair23Lower.AsInt64()).AsInt32(); + Vector256 quad2 = Avx2.UnpackLow(pair01Upper.AsInt64(), pair23Upper.AsInt64()).AsInt32(); + Vector256 quad3 = Avx2.UnpackHigh(pair01Upper.AsInt64(), pair23Upper.AsInt64()).AsInt32(); + return (quad0 + quad1) + (quad2 + quad3); + } + + /// + /// Rotates an 8x8 packed sample block counter-clockwise by transposing and reversing its rows. + /// + /// The source rows, replaced by the rotated rows. + private static void ReverseTranspose(ref InlineArray8> lines) + { + Vector128 pair01Lower = Vector128_.UnpackLow(lines[0], lines[1]); + Vector128 pair23Lower = Vector128_.UnpackLow(lines[2], lines[3]); + Vector128 pair01Upper = Vector128_.UnpackHigh(lines[0], lines[1]); + Vector128 pair23Upper = Vector128_.UnpackHigh(lines[2], lines[3]); + Vector128 pair45Lower = Vector128_.UnpackLow(lines[4], lines[5]); + Vector128 pair67Lower = Vector128_.UnpackLow(lines[6], lines[7]); + Vector128 pair45Upper = Vector128_.UnpackHigh(lines[4], lines[5]); + Vector128 pair67Upper = Vector128_.UnpackHigh(lines[6], lines[7]); + Vector128 quad03Lower = Vector128_.UnpackLow(pair01Lower.AsInt32(), pair23Lower.AsInt32()); + Vector128 quad47Lower = Vector128_.UnpackLow(pair45Lower.AsInt32(), pair67Lower.AsInt32()); + Vector128 quad03Middle = Vector128_.UnpackHigh(pair01Lower.AsInt32(), pair23Lower.AsInt32()); + Vector128 quad47Middle = Vector128_.UnpackHigh(pair45Lower.AsInt32(), pair67Lower.AsInt32()); + Vector128 quad03Upper = Vector128_.UnpackLow(pair01Upper.AsInt32(), pair23Upper.AsInt32()); + Vector128 quad47Upper = Vector128_.UnpackLow(pair45Upper.AsInt32(), pair67Upper.AsInt32()); + Vector128 quad03Highest = Vector128_.UnpackHigh(pair01Upper.AsInt32(), pair23Upper.AsInt32()); + Vector128 quad47Highest = Vector128_.UnpackHigh(pair45Upper.AsInt32(), pair67Upper.AsInt32()); + + // Writing in reverse order turns the normal transpose into the counter-clockwise rotation required to reuse + // the same four-direction cost kernel for the orthogonal half of the search. + lines[7] = Vector128_.UnpackLow(quad03Lower.AsInt64(), quad47Lower.AsInt64()).AsInt16(); + lines[6] = Vector128_.UnpackHigh(quad03Lower.AsInt64(), quad47Lower.AsInt64()).AsInt16(); + lines[5] = Vector128_.UnpackLow(quad03Middle.AsInt64(), quad47Middle.AsInt64()).AsInt16(); + lines[4] = Vector128_.UnpackHigh(quad03Middle.AsInt64(), quad47Middle.AsInt64()).AsInt16(); + lines[3] = Vector128_.UnpackLow(quad03Upper.AsInt64(), quad47Upper.AsInt64()).AsInt16(); + lines[2] = Vector128_.UnpackHigh(quad03Upper.AsInt64(), quad47Upper.AsInt64()).AsInt16(); + lines[1] = Vector128_.UnpackLow(quad03Highest.AsInt64(), quad47Highest.AsInt64()).AsInt16(); + lines[0] = Vector128_.UnpackHigh(quad03Highest.AsInt64(), quad47Highest.AsInt64()).AsInt16(); + } + + /// + /// Rotates two packed 8x8 sample blocks counter-clockwise within their independent 128-bit lanes. + /// + /// The source rows for both blocks, replaced by the rotated rows. + private static void ReverseTranspose(ref InlineArray8> lines) + { + Vector256 pair01Lower = Avx2.UnpackLow(lines[0], lines[1]); + Vector256 pair23Lower = Avx2.UnpackLow(lines[2], lines[3]); + Vector256 pair01Upper = Avx2.UnpackHigh(lines[0], lines[1]); + Vector256 pair23Upper = Avx2.UnpackHigh(lines[2], lines[3]); + Vector256 pair45Lower = Avx2.UnpackLow(lines[4], lines[5]); + Vector256 pair67Lower = Avx2.UnpackLow(lines[6], lines[7]); + Vector256 pair45Upper = Avx2.UnpackHigh(lines[4], lines[5]); + Vector256 pair67Upper = Avx2.UnpackHigh(lines[6], lines[7]); + Vector256 quad03Lower = Avx2.UnpackLow(pair01Lower.AsInt32(), pair23Lower.AsInt32()); + Vector256 quad47Lower = Avx2.UnpackLow(pair45Lower.AsInt32(), pair67Lower.AsInt32()); + Vector256 quad03Middle = Avx2.UnpackHigh(pair01Lower.AsInt32(), pair23Lower.AsInt32()); + Vector256 quad47Middle = Avx2.UnpackHigh(pair45Lower.AsInt32(), pair67Lower.AsInt32()); + Vector256 quad03Upper = Avx2.UnpackLow(pair01Upper.AsInt32(), pair23Upper.AsInt32()); + Vector256 quad47Upper = Avx2.UnpackLow(pair45Upper.AsInt32(), pair67Upper.AsInt32()); + Vector256 quad03Highest = Avx2.UnpackHigh(pair01Upper.AsInt32(), pair23Upper.AsInt32()); + Vector256 quad47Highest = Avx2.UnpackHigh(pair45Upper.AsInt32(), pair67Upper.AsInt32()); + + lines[7] = Avx2.UnpackLow(quad03Lower.AsInt64(), quad47Lower.AsInt64()).AsInt16(); + lines[6] = Avx2.UnpackHigh(quad03Lower.AsInt64(), quad47Lower.AsInt64()).AsInt16(); + lines[5] = Avx2.UnpackLow(quad03Middle.AsInt64(), quad47Middle.AsInt64()).AsInt16(); + lines[4] = Avx2.UnpackHigh(quad03Middle.AsInt64(), quad47Middle.AsInt64()).AsInt16(); + lines[3] = Avx2.UnpackLow(quad03Upper.AsInt64(), quad47Upper.AsInt64()).AsInt16(); + lines[2] = Avx2.UnpackHigh(quad03Upper.AsInt64(), quad47Upper.AsInt64()).AsInt16(); + lines[1] = Avx2.UnpackLow(quad03Highest.AsInt64(), quad47Highest.AsInt64()).AsInt16(); + lines[0] = Avx2.UnpackHigh(quad03Highest.AsInt64(), quad47Highest.AsInt64()).AsInt16(); + } + + /// + /// Finds the dominant direction without vector instructions or a frame-block partial-sum buffer. + /// + /// The bordered, deblocked source plane. + /// The offset of the block's top-left sample. + /// The number of samples between adjacent source rows. + /// The number of bits above the eight-bit analysis precision. + /// Receives the variance difference between the selected and orthogonal directions. + /// The zero-based AV1 direction index. + private static int FindDirectionScalar( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + int coefficientShift, + out int variance) + { + InlineArray8 costs = default; + InlineArray16 partials = default; + Span lineSums = partials; + + // The fallback reuses one 15-line accumulator for each direction. Re-reading the 8x8 block is preferable to + // reserving and clearing the old 120-element partial table on every block when SIMD is explicitly disabled. + for (int direction = 0; direction < 8; direction++) + { + lineSums.Clear(); + for (int row = 0; row < 8; row++) + { + for (int column = 0; column < 8; column++) + { + int value = (source[sourceOffset + (row * sourceStride) + column] >> coefficientShift) - 128; + lineSums[GetDirectionLine(direction, row, column)] += value; + } + } + + costs[direction] = CalculateDirectionCost(direction, lineSums); + } + + return SelectDirection(ref costs, out variance); + } + + /// + /// Selects the first maximum direction cost and derives its orthogonal variance difference. + /// + /// The eight direction costs in ascending direction order. + /// Receives the scaled difference from the selected direction's orthogonal cost. + /// The zero-based AV1 direction index. + private static int SelectDirection(ref InlineArray8 costs, out int variance) + { + int bestCost = 0; + int bestDirection = 0; + for (int direction = 0; direction < 8; direction++) + { + if (costs[direction] > bestCost) + { + bestCost = costs[direction]; + bestDirection = direction; + } + } + + // All directions omit the same sum-of-squares term, so the scaled difference from the orthogonal cost is the + // directional variance consumed by the luma strength adjustment. + variance = (bestCost - costs[(bestDirection + 4) & 7]) >> 10; + return bestDirection; + } + + /// + /// Maps one source coordinate to its line in the requested AV1 direction. + /// + /// The zero-based AV1 direction index. + /// The source row within the 8x8 block. + /// The source column within the 8x8 block. + /// The zero-based line index. + private static int GetDirectionLine(int direction, int row, int column) => direction switch + { + 0 => row + column, + 1 => row + (column >> 1), + 2 => row, + 3 => 3 + row - (column >> 1), + 4 => 7 + row - column, + 5 => 3 - (row >> 1) + column, + 6 => column, + _ => (row >> 1) + column + }; + + /// + /// Computes one weighted direction cost from its accumulated line sums. + /// + /// The zero-based AV1 direction index. + /// The accumulated line sums. + /// The weighted direction cost. + private static int CalculateDirectionCost(int direction, ReadOnlySpan lineSums) + { + int cost = 0; + if (direction is 2 or 6) + { + for (int line = 0; line < 8; line++) + { + cost += lineSums[line] * lineSums[line]; + } + + return cost * 105; + } + + if ((direction & 1) == 0) + { + for (int line = 0; line < 7; line++) + { + int mirroredLine = 14 - line; + cost += ((lineSums[line] * lineSums[line]) + (lineSums[mirroredLine] * lineSums[mirroredLine])) * GetDivisionMultiplier(line + 1); + } + + return cost + (lineSums[7] * lineSums[7] * 105); + } + + for (int line = 3; line < 8; line++) + { + cost += lineSums[line] * lineSums[line]; + } + + cost *= 105; + for (int line = 0; line < 3; line++) + { + int mirroredLine = 10 - line; + cost += ((lineSums[line] * lineSums[line]) + (lineSums[mirroredLine] * lineSums[mirroredLine])) * GetDivisionMultiplier((line * 2) + 2); + } + + return cost; + } + + /// + /// Gets the common multiple used to compare lines of different lengths without division. + /// + /// The number of samples contributing to the line. + /// The multiplier equal to 840 divided by the line length. + private static int GetDivisionMultiplier(int lineLength) => lineLength switch + { + 1 => 840, + 2 => 420, + 3 => 280, + 4 => 210, + 5 => 168, + 6 => 140, + 7 => 120, + _ => 105 + }; + + /// + /// Applies one packed CDEF kernel with each signed 16-bit lane representing a column in the current output row. + /// + /// The destination sample storage type. + /// The storage-specific output operator. + /// The enabled directional-tap operator. + /// The first element in the bordered source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The first element in the destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + private static void FilterBlockVector( + ref ushort source, + int sourceOffset, + int sourceStride, + ref TSample destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + where TSample : unmanaged + where TOutputOperator : struct, IOutputOperator + where TFilterOperator : struct, IFilterOperator + { + bool clippingRequired = TFilterOperator.EnablePrimary && TFilterOperator.EnableSecondary; + int primaryDampingShift = TFilterOperator.EnablePrimary ? Math.Max(0, primaryDamping - Av1Math.MostSignificantBit((uint)primaryStrength)) : 0; + int secondaryDampingShift = TFilterOperator.EnableSecondary ? Math.Max(0, secondaryDamping - Av1Math.MostSignificantBit((uint)secondaryStrength)) : 0; + int primaryTapSet = (primaryStrength >> coefficientShift) & 1; + int primaryNearOffset = TFilterOperator.EnablePrimary ? GetDirectionOffset(direction, 0, sourceStride) : 0; + int primaryFarOffset = TFilterOperator.EnablePrimary ? GetDirectionOffset(direction, 1, sourceStride) : 0; + int secondaryNearOffset0 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 2) & 7, 0, sourceStride) : 0; + int secondaryFarOffset0 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 2) & 7, 1, sourceStride) : 0; + int secondaryNearOffset1 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 6) & 7, 0, sourceStride) : 0; + int secondaryFarOffset1 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 6) & 7, 1, sourceStride) : 0; + Vector128 primaryNearWeight = Vector128.Create((short)(primaryTapSet == 0 ? 4 : 3)); + Vector128 primaryFarWeight = Vector128.Create((short)(primaryTapSet == 0 ? 2 : 3)); + Vector128 secondaryNearWeight = Vector128.Create((short)2); + Vector128 secondaryFarWeight = Vector128.Create((short)1); + Vector128 sentinel = Vector128.Create((short)VeryLarge); + Vector128 zero = Vector128.Zero; + Vector128 rounding = Vector128.Create((short)8); + Vector128 one = Vector128.Create((short)1); + + for (int row = 0; row < blockHeight; row++) + { + int sourceIndex = sourceOffset + (row * sourceStride); + Vector128 sample = LoadSamples(ref source, sourceIndex, blockWidth); + Vector128 sum = zero; + Vector128 minimum = sample; + Vector128 maximum = sample; + + for (int tap = 0; tap < 2; tap++) + { + if (TFilterOperator.EnablePrimary) + { + int offset = tap == 0 ? primaryNearOffset : primaryFarOffset; + Vector128 neighbor0 = LoadSamples(ref source, sourceIndex + offset, blockWidth); + Vector128 neighbor1 = LoadSamples(ref source, sourceIndex - offset, blockWidth); + Vector128 constrained = Constrain(neighbor0, sample, primaryStrength, primaryDampingShift) + + Constrain(neighbor1, sample, primaryStrength, primaryDampingShift); + + sum += constrained * (tap == 0 ? primaryNearWeight : primaryFarWeight); + if (clippingRequired) + { + minimum = Vector128.Min(minimum, Vector128.Min(neighbor0, neighbor1)); + maximum = MaximumIgnoringSentinel(maximum, neighbor0, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor1, sentinel); + } + } + + if (TFilterOperator.EnableSecondary) + { + int offset0 = tap == 0 ? secondaryNearOffset0 : secondaryFarOffset0; + int offset1 = tap == 0 ? secondaryNearOffset1 : secondaryFarOffset1; + Vector128 neighbor0 = LoadSamples(ref source, sourceIndex + offset0, blockWidth); + Vector128 neighbor1 = LoadSamples(ref source, sourceIndex - offset0, blockWidth); + Vector128 neighbor2 = LoadSamples(ref source, sourceIndex + offset1, blockWidth); + Vector128 neighbor3 = LoadSamples(ref source, sourceIndex - offset1, blockWidth); + Vector128 constrained = Constrain(neighbor0, sample, secondaryStrength, secondaryDampingShift) + + Constrain(neighbor1, sample, secondaryStrength, secondaryDampingShift) + + Constrain(neighbor2, sample, secondaryStrength, secondaryDampingShift) + + Constrain(neighbor3, sample, secondaryStrength, secondaryDampingShift); + + sum += constrained * (tap == 0 ? secondaryNearWeight : secondaryFarWeight); + if (clippingRequired) + { + minimum = Vector128.Min(minimum, Vector128.Min(Vector128.Min(neighbor0, neighbor1), Vector128.Min(neighbor2, neighbor3))); + maximum = MaximumIgnoringSentinel(maximum, neighbor0, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor1, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor2, sentinel); + maximum = MaximumIgnoringSentinel(maximum, neighbor3, sentinel); + } + } + } + + // The sign lane contributes the one-unit correction required by AV1's asymmetric rounding for negative sums. + Vector128 correction = (sum >> 15) & one; + Vector128 filtered = sample + ((sum + rounding - correction) >> 4); + if (clippingRequired) + { + filtered = Vector128.Min(Vector128.Max(filtered, minimum), maximum); + } + + TOutputOperator.StoreVector(ref destination, destinationOffset + (row * destinationStride), filtered, blockWidth); + } + } + + /// + /// Applies the scalar CDEF fallback to one block. + /// + /// The destination sample storage type. + /// The storage-specific output operator. + /// The enabled directional-tap operator. + /// The first element in the bordered source plane. + /// The offset of the block's top-left source sample. + /// The number of samples between adjacent source rows. + /// The first element in the destination plane. + /// The offset of the block's top-left destination sample. + /// The number of samples between adjacent destination rows. + /// The bit-depth-scaled primary strength. + /// The bit-depth-scaled secondary strength. + /// The zero-based AV1 direction index. + /// The damping value applied to primary taps. + /// The damping value applied to secondary taps. + /// The number of bits above eight-bit sample precision. + /// The block width in plane samples. + /// The block height in plane samples. + private static void FilterBlockScalar( + ref ushort source, + int sourceOffset, + int sourceStride, + ref TSample destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int primaryDamping, + int secondaryDamping, + int coefficientShift, + int blockWidth, + int blockHeight) + where TSample : unmanaged + where TOutputOperator : struct, IOutputOperator + where TFilterOperator : struct, IFilterOperator + { + bool clippingRequired = TFilterOperator.EnablePrimary && TFilterOperator.EnableSecondary; + int primaryDampingShift = TFilterOperator.EnablePrimary ? Math.Max(0, primaryDamping - Av1Math.MostSignificantBit((uint)primaryStrength)) : 0; + int secondaryDampingShift = TFilterOperator.EnableSecondary ? Math.Max(0, secondaryDamping - Av1Math.MostSignificantBit((uint)secondaryStrength)) : 0; + int primaryTapSet = (primaryStrength >> coefficientShift) & 1; + int primaryNearOffset = TFilterOperator.EnablePrimary ? GetDirectionOffset(direction, 0, sourceStride) : 0; + int primaryFarOffset = TFilterOperator.EnablePrimary ? GetDirectionOffset(direction, 1, sourceStride) : 0; + int secondaryNearOffset0 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 2) & 7, 0, sourceStride) : 0; + int secondaryFarOffset0 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 2) & 7, 1, sourceStride) : 0; + int secondaryNearOffset1 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 6) & 7, 0, sourceStride) : 0; + int secondaryFarOffset1 = TFilterOperator.EnableSecondary ? GetDirectionOffset((direction + 6) & 7, 1, sourceStride) : 0; + + for (int row = 0; row < blockHeight; row++) + { + for (int column = 0; column < blockWidth; column++) + { + int sourceIndex = sourceOffset + (row * sourceStride) + column; + int sample = Unsafe.Add(ref source, sourceIndex); + int sum = 0; + int minimum = sample; + int maximum = sample; + + for (int tap = 0; tap < 2; tap++) + { + if (TFilterOperator.EnablePrimary) + { + int offset = tap == 0 ? primaryNearOffset : primaryFarOffset; + int neighbor0 = Unsafe.Add(ref source, sourceIndex + offset); + int neighbor1 = Unsafe.Add(ref source, sourceIndex - offset); + int weight = primaryTapSet == 0 ? (tap == 0 ? 4 : 2) : 3; + sum += weight * Constrain(neighbor0 - sample, primaryStrength, primaryDampingShift); + sum += weight * Constrain(neighbor1 - sample, primaryStrength, primaryDampingShift); + + if (clippingRequired) + { + maximum = neighbor0 != VeryLarge ? Math.Max(maximum, neighbor0) : maximum; + maximum = neighbor1 != VeryLarge ? Math.Max(maximum, neighbor1) : maximum; + minimum = Math.Min(minimum, Math.Min(neighbor0, neighbor1)); + } + } + + if (TFilterOperator.EnableSecondary) + { + int offset0 = tap == 0 ? secondaryNearOffset0 : secondaryFarOffset0; + int offset1 = tap == 0 ? secondaryNearOffset1 : secondaryFarOffset1; + int neighbor0 = Unsafe.Add(ref source, sourceIndex + offset0); + int neighbor1 = Unsafe.Add(ref source, sourceIndex - offset0); + int neighbor2 = Unsafe.Add(ref source, sourceIndex + offset1); + int neighbor3 = Unsafe.Add(ref source, sourceIndex - offset1); + int weight = tap == 0 ? 2 : 1; + sum += weight * Constrain(neighbor0 - sample, secondaryStrength, secondaryDampingShift); + sum += weight * Constrain(neighbor1 - sample, secondaryStrength, secondaryDampingShift); + sum += weight * Constrain(neighbor2 - sample, secondaryStrength, secondaryDampingShift); + sum += weight * Constrain(neighbor3 - sample, secondaryStrength, secondaryDampingShift); + + if (clippingRequired) + { + maximum = neighbor0 != VeryLarge ? Math.Max(maximum, neighbor0) : maximum; + maximum = neighbor1 != VeryLarge ? Math.Max(maximum, neighbor1) : maximum; + maximum = neighbor2 != VeryLarge ? Math.Max(maximum, neighbor2) : maximum; + maximum = neighbor3 != VeryLarge ? Math.Max(maximum, neighbor3) : maximum; + minimum = Math.Min(minimum, Math.Min(Math.Min(neighbor0, neighbor1), Math.Min(neighbor2, neighbor3))); + } + } + } + + int filtered = sample + ((8 + sum - (sum < 0 ? 1 : 0)) >> 4); + TOutputOperator.StoreScalar( + ref destination, + destinationOffset + (row * destinationStride) + column, + clippingRequired ? Av1Math.Clip3(minimum, maximum, filtered) : filtered); + } + } + } + + /// + /// Loads four or eight contiguous source samples into signed 16-bit lanes. + /// + /// The first element in the bordered source plane. + /// The offset of the first sample to load. + /// The number of valid samples. + /// The samples in the low vector lanes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadSamples(ref ushort source, int offset, int count) + { + if (count == 8) + { + return Vector128.LoadUnsafe(ref source, (nuint)offset).AsInt16(); + } + + // Four-wide chroma blocks load exactly 64 bits so the final block never reads beyond its two-sample sentinel border. + ref byte sourceBytes = ref Unsafe.As(ref Unsafe.Add(ref source, offset)); + return Vector128.Create(Unsafe.ReadUnaligned(ref sourceBytes), 0UL).AsInt16(); + } + + /// + /// Loads two 8-wide rows or four 4-wide rows into the independent row groups of one 256-bit vector. + /// + /// The first element in the bordered source plane. + /// The offset of the first sample to load. + /// The number of samples between adjacent rows. + /// The number of valid samples in each row. + /// The packed source rows. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 LoadRows(ref ushort source, int offset, int stride, int width) + { + if (width == 8) + { + Vector128 firstRow = Vector128.LoadUnsafe(ref source, (nuint)offset).AsInt16(); + Vector128 secondRow = Vector128.LoadUnsafe(ref source, (nuint)(offset + stride)).AsInt16(); + return Vector256.Create(firstRow, secondRow); + } + + // Four-row batches use one 64-bit load per row. Pairing two rows in each 128-bit lane preserves the exact row + // boundaries required by libaom's lane-local shifts while avoiding reads beyond the frame sentinel border. + Vector64 row0 = LoadSamples(ref source, offset, 4).GetLower(); + Vector64 row1 = LoadSamples(ref source, offset + stride, 4).GetLower(); + Vector64 row2 = LoadSamples(ref source, offset + (2 * stride), 4).GetLower(); + Vector64 row3 = LoadSamples(ref source, offset + (3 * stride), 4).GetLower(); + return Vector256.Create(Vector128.Create(row0, row1), Vector128.Create(row2, row3)); + } + + /// + /// Stores two 8-wide rows or four 4-wide rows from one packed result vector. + /// + /// The destination sample storage type. + /// The storage-specific output operator. + /// The first element in the destination plane. + /// The offset of the first row to write. + /// The number of samples between adjacent destination rows. + /// The packed filtered rows. + /// The number of valid samples in each row. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreRows( + ref TSample destination, + int offset, + int stride, + Vector256 value, + int width) + where TSample : unmanaged + where TOutputOperator : struct, IOutputOperator + { + Vector128 lower = value.GetLower(); + Vector128 upper = value.GetUpper(); + if (width == 8) + { + TOutputOperator.StoreVector(ref destination, offset, lower, 8); + TOutputOperator.StoreVector(ref destination, offset + stride, upper, 8); + return; + } + + TOutputOperator.StoreVector(ref destination, offset, lower, 4); + TOutputOperator.StoreVector(ref destination, offset + stride, Vector128.Create(lower.GetUpper(), Vector64.Zero), 4); + TOutputOperator.StoreVector(ref destination, offset + (2 * stride), upper, 4); + TOutputOperator.StoreVector(ref destination, offset + (3 * stride), Vector128.Create(upper.GetUpper(), Vector64.Zero), 4); + } + + /// + /// Limits packed neighbor differences according to one filter strength and its pre-adjusted damping shift. + /// + /// The neighboring samples. + /// The current samples. + /// The bit-depth-scaled filter strength. + /// The damping shift after accounting for the threshold magnitude. + /// The signed constrained differences. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Constrain(Vector128 neighbor, Vector128 sample, int threshold, int dampingShift) + { + Vector128 difference = neighbor - sample; + Vector128 sign = difference >> 15; + Vector128 magnitude = Vector128.Abs(difference).AsUInt16(); + Vector128 remaining = Vector128.SubtractSaturate(Vector128.Create((ushort)threshold), magnitude >> dampingShift); + Vector128 constrained = Vector128.Min(magnitude, remaining).AsInt16(); + + // Adding the all-bits sign before XOR reproduces sign(value) * magnitude without a branch or a multiply. + return (constrained + sign) ^ sign; + } + + /// + /// Limits packed neighbor differences for several independent output rows. + /// + /// The neighboring samples. + /// The current samples. + /// The bit-depth-scaled filter strength. + /// The damping shift after accounting for the threshold magnitude. + /// The signed constrained differences. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Constrain(Vector256 neighbor, Vector256 sample, int threshold, int dampingShift) + { + Vector256 difference = neighbor - sample; + Vector256 sign = difference >> 15; + Vector256 magnitude = Vector256.Abs(difference).AsUInt16(); + Vector256 remaining = Vector256.SubtractSaturate(Vector256.Create((ushort)threshold), magnitude >> dampingShift); + Vector256 constrained = Vector256.Min(magnitude, remaining).AsInt16(); + return (constrained + sign) ^ sign; + } + + /// + /// Updates a packed maximum while treating unavailable-neighbor sentinels as zero. + /// + /// The current per-lane maximum. + /// The candidate neighboring samples. + /// The unavailable-neighbor sentinel in every lane. + /// The updated per-lane maximum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MaximumIgnoringSentinel(Vector128 maximum, Vector128 candidate, Vector128 sentinel) + { + Vector128 available = Vector128.ConditionalSelect(Vector128.Equals(candidate, sentinel), Vector128.Zero, candidate); + return Vector128.Max(maximum, available); + } + + /// + /// Updates packed row maxima while treating unavailable-neighbor sentinels as zero. + /// + /// The current per-lane maximum. + /// The candidate neighboring samples. + /// The unavailable-neighbor sentinel in every lane. + /// The updated per-lane maximum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MaximumIgnoringSentinel(Vector256 maximum, Vector256 candidate, Vector256 sentinel) + { + Vector256 available = Vector256.ConditionalSelect(Vector256.Equals(candidate, sentinel), Vector256.Zero, candidate); + return Vector256.Max(maximum, available); + } + + /// + /// Limits one scalar neighbor difference according to a filter strength and damping value. + /// + /// The signed difference from the current sample. + /// The bit-depth-scaled filter strength. + /// The damping shift after accounting for the threshold magnitude. + /// The signed constrained difference. + private static int Constrain(int difference, int threshold, int dampingShift) + { + int magnitude = Math.Abs(difference); + int constrained = Av1Math.Clip3(0, magnitude, threshold - (magnitude >> dampingShift)); + return difference < 0 ? -constrained : constrained; + } + + /// + /// Converts a direction and tap number to a signed plane-buffer offset. + /// + /// The zero-based AV1 direction index. + /// The zero-based distance index. + /// The number of samples between adjacent rows. + /// The signed sample offset. + private static int GetDirectionOffset(int direction, int tap, int stride) => direction switch + { + 0 => tap == 0 ? -stride + 1 : (-2 * stride) + 2, + 1 => tap == 0 ? 1 : -stride + 2, + 2 => tap == 0 ? 1 : 2, + 3 => tap == 0 ? 1 : stride + 2, + 4 => tap == 0 ? stride + 1 : (2 * stride) + 2, + 5 => tap == 0 ? stride : (2 * stride) + 1, + 6 => tap == 0 ? stride : 2 * stride, + _ => tap == 0 ? stride : (2 * stride) - 1 + }; + + /// + /// Enables both directional tap groups and their combined clipping rule. + /// + private readonly struct PrimaryAndSecondaryFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => true; + + /// + public static bool EnableSecondary => true; + } + + /// + /// Enables only the primary directional taps. + /// + private readonly struct PrimaryFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => true; + + /// + public static bool EnableSecondary => false; + } + + /// + /// Enables only the secondary off-axis taps. + /// + private readonly struct SecondaryFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => false; + + /// + public static bool EnableSecondary => true; + } + + /// + /// Disables both tap groups so the source block is copied unchanged. + /// + private readonly struct CopyFilterOperator : IFilterOperator + { + /// + public static bool EnablePrimary => false; + + /// + public static bool EnableSecondary => false; + } + + /// + /// Writes filtered samples to eight-bit plane storage. + /// + private readonly struct ByteOutputOperator : IOutputOperator + { + /// + public static void StoreVector(ref byte destination, int offset, Vector128 value, int count) + { + Vector64 packed = Vector128.Narrow(value.AsUInt16(), Vector128.Zero).GetLower(); + ref byte output = ref Unsafe.Add(ref destination, offset); + if (count == 8) + { + packed.StoreUnsafe(ref output); + } + else + { + Unsafe.WriteUnaligned(ref output, packed.AsUInt32().ToScalar()); + } + } + + /// + public static void StoreScalar(ref byte destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (byte)value; + } + + /// + /// Writes filtered samples to 16-bit plane storage. + /// + private readonly struct UInt16OutputOperator : IOutputOperator + { + /// + public static void StoreVector(ref ushort destination, int offset, Vector128 value, int count) + { + ref ushort output = ref Unsafe.Add(ref destination, offset); + if (count == 8) + { + value.AsUInt16().StoreUnsafe(ref output); + } + else + { + ref byte outputBytes = ref Unsafe.As(ref output); + Unsafe.WriteUnaligned(ref outputBytes, value.AsUInt64().GetLower().ToScalar()); + } + } + + /// + public static void StoreScalar(ref ushort destination, int offset, int value) => Unsafe.Add(ref destination, offset) = (ushort)value; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefKernels.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefKernels.cs deleted file mode 100644 index d0cbf9ef2..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefKernels.cs +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; - -/// -/// Provides the scalar constrained directional enhancement filter operations defined by AV1. -/// -internal static class Av1CdefKernels -{ - /// - /// The sample value used for neighbors outside the coded frame. - /// - public const ushort VeryLarge = 0x4000; - - /// - /// The horizontal offsets for the nearest primary or secondary taps in each direction. - /// - private static readonly int[] DirectionX1 = [1, 1, 1, 1, 1, 0, 0, 0]; - - /// - /// The vertical offsets for the nearest primary or secondary taps in each direction. - /// - private static readonly int[] DirectionY1 = [-1, 0, 0, 0, 1, 1, 1, 1]; - - /// - /// The horizontal offsets for the furthest primary or secondary taps in each direction. - /// - private static readonly int[] DirectionX2 = [2, 2, 2, 2, 2, 1, 0, -1]; - - /// - /// The vertical offsets for the furthest primary or secondary taps in each direction. - /// - private static readonly int[] DirectionY2 = [-2, -1, 0, 1, 2, 2, 2, 2]; - - /// - /// The primary-tap weights selected by the parity of the unscaled primary strength. - /// - private static readonly int[,] PrimaryTaps = - { - { 4, 2 }, - { 3, 3 } - }; - - /// - /// The secondary-tap weights for the nearest and furthest samples. - /// - private static readonly int[] SecondaryTaps = [2, 1]; - - /// - /// The common multiples used to compare line variance without division. - /// - private static readonly int[] DivisionTable = [0, 840, 420, 280, 210, 168, 140, 120, 105]; - - /// - /// The direction mapping for horizontally subsampled, vertically full-resolution chroma. - /// - private static readonly int[] DirectionMap422 = [7, 0, 2, 4, 5, 6, 6, 6]; - - /// - /// The direction mapping for horizontally full-resolution, vertically subsampled chroma. - /// - private static readonly int[] DirectionMap440 = [1, 2, 2, 2, 3, 4, 6, 0]; - - /// - /// Finds the dominant direction of an 8x8 luma block and its directional variance. - /// - /// The bordered, deblocked source plane. - /// The offset of the block's top-left sample. - /// The number of samples between adjacent source rows. - /// The number of bits above the eight-bit analysis precision. - /// Receives the variance difference between the selected and orthogonal directions. - /// The zero-based AV1 direction index. - public static int FindDirection( - ReadOnlySpan source, - int sourceOffset, - int sourceStride, - int coefficientShift, - out int variance) - { - Span partial = stackalloc int[8 * 15]; - Span cost = stackalloc int[8]; - partial.Clear(); - cost.Clear(); - - for (int row = 0; row < 8; row++) - { - for (int column = 0; column < 8; column++) - { - // Direction analysis deliberately reduces every source to eight-bit precision so its - // strength selection is identical for 8-, 10-, and 12-bit coded images. - int value = (source[sourceOffset + (row * sourceStride) + column] >> coefficientShift) - 128; - partial[(0 * 15) + row + column] += value; - partial[(1 * 15) + row + (column / 2)] += value; - partial[(2 * 15) + row] += value; - partial[(3 * 15) + 3 + row - (column / 2)] += value; - partial[(4 * 15) + 7 + row - column] += value; - partial[(5 * 15) + 3 - (row / 2) + column] += value; - partial[(6 * 15) + column] += value; - partial[(7 * 15) + (row / 2) + column] += value; - } - } - - for (int i = 0; i < 8; i++) - { - cost[2] += partial[(2 * 15) + i] * partial[(2 * 15) + i]; - cost[6] += partial[(6 * 15) + i] * partial[(6 * 15) + i]; - } - - cost[2] *= DivisionTable[8]; - cost[6] *= DivisionTable[8]; - for (int i = 0; i < 7; i++) - { - cost[0] += ((partial[(0 * 15) + i] * partial[(0 * 15) + i]) + - (partial[(0 * 15) + 14 - i] * partial[(0 * 15) + 14 - i])) * DivisionTable[i + 1]; - cost[4] += ((partial[(4 * 15) + i] * partial[(4 * 15) + i]) + - (partial[(4 * 15) + 14 - i] * partial[(4 * 15) + 14 - i])) * DivisionTable[i + 1]; - } - - cost[0] += partial[(0 * 15) + 7] * partial[(0 * 15) + 7] * DivisionTable[8]; - cost[4] += partial[(4 * 15) + 7] * partial[(4 * 15) + 7] * DivisionTable[8]; - for (int direction = 1; direction < 8; direction += 2) - { - for (int i = 0; i < 5; i++) - { - cost[direction] += partial[(direction * 15) + 3 + i] * partial[(direction * 15) + 3 + i]; - } - - cost[direction] *= DivisionTable[8]; - for (int i = 0; i < 3; i++) - { - cost[direction] += ((partial[(direction * 15) + i] * partial[(direction * 15) + i]) + - (partial[(direction * 15) + 10 - i] * partial[(direction * 15) + 10 - i])) * DivisionTable[(2 * i) + 2]; - } - } - - int bestCost = 0; - int bestDirection = 0; - for (int direction = 0; direction < 8; direction++) - { - if (cost[direction] > bestCost) - { - bestCost = cost[direction]; - bestDirection = direction; - } - } - - // Both costs omit the same sum-of-squares term. Their scaled difference is the - // directional variance consumed by AV1's luma strength adjustment. - variance = (bestCost - cost[(bestDirection + 4) & 7]) >> 10; - return bestDirection; - } - - /// - /// Adjusts a luma primary strength according to the directional variance of its 8x8 block. - /// - /// The bit-depth-scaled primary strength. - /// The directional variance returned by . - /// The variance-adjusted primary strength. - public static int AdjustStrength(int strength, int variance) - { - int varianceClass = variance >> 6; - int adjustment = varianceClass != 0 ? Math.Min(Av1Math.MostSignificantBit((uint)varianceClass), 12) : 0; - return variance != 0 ? ((strength * (4 + adjustment)) + 8) >> 4 : 0; - } - - /// - /// Converts a luma direction to the matching chroma direction for asymmetric subsampling. - /// - /// The zero-based luma direction index. - /// The horizontal chroma subsampling shift. - /// The vertical chroma subsampling shift. - /// The direction index in the chroma sample grid. - public static int ConvertDirection(int direction, int subsamplingX, int subsamplingY) - { - if (subsamplingX == subsamplingY) - { - return direction; - } - - return subsamplingX != 0 ? DirectionMap422[direction] : DirectionMap440[direction]; - } - - /// - /// Filters one luma or chroma block from an immutable bordered source plane. - /// - /// The bordered, deblocked source plane. - /// The offset of the block's top-left source sample. - /// The number of samples between adjacent source rows. - /// The unbordered filtered destination plane. - /// The offset of the block's top-left destination sample. - /// The number of samples between adjacent destination rows. - /// The bit-depth-scaled primary strength. - /// The bit-depth-scaled secondary strength. - /// The zero-based AV1 direction index. - /// The damping value applied to primary taps. - /// The damping value applied to secondary taps. - /// The number of bits above eight-bit sample precision. - /// The block width in plane samples. - /// The block height in plane samples. - public static void FilterBlock( - ReadOnlySpan source, - int sourceOffset, - int sourceStride, - Span destination, - int destinationOffset, - int destinationStride, - int primaryStrength, - int secondaryStrength, - int direction, - int primaryDamping, - int secondaryDamping, - int coefficientShift, - int blockWidth, - int blockHeight) - { - bool enablePrimary = primaryStrength != 0; - bool enableSecondary = secondaryStrength != 0; - bool clippingRequired = enablePrimary && enableSecondary; - int primaryTapSet = (primaryStrength >> coefficientShift) & 1; - - for (int row = 0; row < blockHeight; row++) - { - for (int column = 0; column < blockWidth; column++) - { - int sourceIndex = sourceOffset + (row * sourceStride) + column; - int sample = source[sourceIndex]; - int sum = 0; - int minimum = sample; - int maximum = sample; - - for (int tap = 0; tap < 2; tap++) - { - if (enablePrimary) - { - int primaryDirectionOffset = GetDirectionOffset(direction, tap, sourceStride); - int neighbor0 = source[sourceIndex + primaryDirectionOffset]; - int neighbor1 = source[sourceIndex - primaryDirectionOffset]; - int weight = PrimaryTaps[primaryTapSet, tap]; - sum += weight * Constrain(neighbor0 - sample, primaryStrength, primaryDamping); - sum += weight * Constrain(neighbor1 - sample, primaryStrength, primaryDamping); - - if (clippingRequired) - { - maximum = neighbor0 != VeryLarge ? Math.Max(maximum, neighbor0) : maximum; - maximum = neighbor1 != VeryLarge ? Math.Max(maximum, neighbor1) : maximum; - minimum = Math.Min(minimum, neighbor0); - minimum = Math.Min(minimum, neighbor1); - } - } - - if (enableSecondary) - { - int secondaryDirection0 = GetDirectionOffset((direction + 2) & 7, tap, sourceStride); - int secondaryDirection1 = GetDirectionOffset((direction + 6) & 7, tap, sourceStride); - int neighbor0 = source[sourceIndex + secondaryDirection0]; - int neighbor1 = source[sourceIndex - secondaryDirection0]; - int neighbor2 = source[sourceIndex + secondaryDirection1]; - int neighbor3 = source[sourceIndex - secondaryDirection1]; - int weight = SecondaryTaps[tap]; - - if (clippingRequired) - { - maximum = neighbor0 != VeryLarge ? Math.Max(maximum, neighbor0) : maximum; - maximum = neighbor1 != VeryLarge ? Math.Max(maximum, neighbor1) : maximum; - maximum = neighbor2 != VeryLarge ? Math.Max(maximum, neighbor2) : maximum; - maximum = neighbor3 != VeryLarge ? Math.Max(maximum, neighbor3) : maximum; - minimum = Math.Min(minimum, neighbor0); - minimum = Math.Min(minimum, neighbor1); - minimum = Math.Min(minimum, neighbor2); - minimum = Math.Min(minimum, neighbor3); - } - - sum += weight * Constrain(neighbor0 - sample, secondaryStrength, secondaryDamping); - sum += weight * Constrain(neighbor1 - sample, secondaryStrength, secondaryDamping); - sum += weight * Constrain(neighbor2 - sample, secondaryStrength, secondaryDamping); - sum += weight * Constrain(neighbor3 - sample, secondaryStrength, secondaryDamping); - } - } - - // The negative-sum correction preserves AV1's asymmetric signed rounding exactly. - int filtered = sample + ((8 + sum - (sum < 0 ? 1 : 0)) >> 4); - destination[destinationOffset + (row * destinationStride) + column] = - (ushort)(clippingRequired ? Av1Math.Clip3(minimum, maximum, filtered) : filtered); - } - } - } - - /// - /// Converts a direction and tap number to a signed plane-buffer offset. - /// - /// The zero-based AV1 direction index. - /// The zero-based distance index. - /// The number of samples between adjacent rows. - /// The signed sample offset. - private static int GetDirectionOffset(int direction, int tap, int stride) - => tap == 0 - ? (DirectionY1[direction] * stride) + DirectionX1[direction] - : (DirectionY2[direction] * stride) + DirectionX2[direction]; - - /// - /// Limits a neighbor difference according to a filter strength and damping value. - /// - /// The signed difference from the current sample. - /// The bit-depth-scaled filter strength. - /// The damping value. - /// The signed constrained difference. - private static int Constrain(int difference, int threshold, int damping) - { - if (threshold == 0) - { - return 0; - } - - // Stronger thresholds reduce the effective damping shift, matching the AV1 constrain function. - int shift = Math.Max(0, damping - Av1Math.MostSignificantBit((uint)threshold)); - int magnitude = Math.Abs(difference); - int constrained = Av1Math.Clip3(0, magnitude, threshold - (magnitude >> shift)); - return difference < 0 ? -constrained : constrained; - } -} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CdefFilterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CdefFilterTests.cs new file mode 100644 index 000000000..d982878b6 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CdefFilterTests.cs @@ -0,0 +1,627 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; +using SixLabors.ImageSharp.Tests.TestUtilities; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies AV1 constrained directional enhancement filtering across sample precision, block geometry, and intrinsic tiers. +/// +[Trait("Format", "Avif")] +public class Av1CdefFilterTests +{ + /// + /// The hardware configurations required to exercise packed filtering and the scalar fallback. + /// + private const HwIntrinsics Configurations = HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic; + + /// + /// The row stride of the bordered source plane used by the filter tests. + /// + private const int SourceStride = 16; + + /// + /// The number of unavailable samples surrounding the test image. + /// + private const int SourceBorder = 2; + + /// + /// Verifies direction selection and variance against an independent scalar definition. + /// + [Fact] + public void FindDirectionMatchesIndependentDefinitionAcrossIntrinsicTiers() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateDirections, Configurations); + + /// + /// Verifies every CDEF block geometry and strength mode against an independent scalar definition. + /// + [Fact] + public void FilterBlockMatchesIndependentDefinitionAcrossIntrinsicTiers() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateFilters, Configurations); + + /// + /// Verifies the complete asymmetric chroma direction mappings and the unchanged symmetric mappings. + /// + [Fact] + public void ConvertDirectionMatchesSubsamplingGeometry() + { + int[] horizontalSubsampling = [7, 0, 2, 4, 5, 6, 6, 6]; + int[] verticalSubsampling = [1, 2, 2, 2, 3, 4, 6, 0]; + for (int direction = 0; direction < 8; direction++) + { + Assert.Equal(horizontalSubsampling[direction], Av1CdefFilter.ConvertDirection(direction, 1, 0)); + Assert.Equal(verticalSubsampling[direction], Av1CdefFilter.ConvertDirection(direction, 0, 1)); + Assert.Equal(direction, Av1CdefFilter.ConvertDirection(direction, 0, 0)); + Assert.Equal(direction, Av1CdefFilter.ConvertDirection(direction, 1, 1)); + } + } + + /// + /// Verifies luma strength adjustment at zero, logarithmic-class boundaries, and the capped variance class. + /// + [Fact] + public void AdjustStrengthMatchesIndependentDefinition() + { + foreach (int strength in new[] { 0, 4, 15, 60 }) + { + foreach (int variance in new[] { 0, 1, 63, 64, 255, 4096, 1 << 20 }) + { + int varianceClass = variance >> 6; + int adjustment = varianceClass == 0 ? 0 : Math.Min(BitOperations.Log2((uint)varianceClass), 12); + int expected = variance == 0 ? 0 : ((strength * (4 + adjustment)) + 8) >> 4; + Assert.Equal(expected, Av1CdefFilter.AdjustStrength(strength, variance)); + } + } + } + + /// + /// Verifies that the deterministic direction corpus exercises every selected-direction branch. + /// + [Fact] + public void DirectionCorpusCoversEveryDirection() + { + const int stride = 32; + const int sourceOffset = (4 * stride) + 5; + int secondSourceOffset = sourceOffset + 8; + HashSet observedDirections = []; + for (int pattern = 0; pattern < 8; pattern++) + { + ushort[] source = new ushort[stride * 16]; + PopulateDirectionSource(source, sourceOffset, stride, pattern, 8); + observedDirections.Add(FindDirectionReference(source, sourceOffset, stride, 0, out _)); + observedDirections.Add(FindDirectionReference(source, secondSourceOffset, stride, 0, out _)); + } + + Assert.Equal(Enumerable.Range(0, 8), observedDirections.Order()); + } + + /// + /// Exercises direction search with multiple source patterns at every supported sample precision. + /// + private static void ValidateDirections() + { + const int stride = 32; + const int sourceOffset = (4 * stride) + 5; + + foreach (int bitDepth in new[] { 8, 10, 12 }) + { + int coefficientShift = bitDepth - 8; + for (int pattern = 0; pattern < 8; pattern++) + { + ushort[] source = new ushort[stride * 16]; + int secondSourceOffset = sourceOffset + 8; + PopulateDirectionSource(source, sourceOffset, stride, pattern, bitDepth); + + int expectedDirection = FindDirectionReference(source, sourceOffset, stride, coefficientShift, out int expectedVariance); + int actualDirection = Av1CdefFilter.FindDirection(source, sourceOffset, stride, coefficientShift, out int actualVariance); + Assert.Equal(expectedDirection, actualDirection); + Assert.Equal(expectedVariance, actualVariance); + + int secondExpectedDirection = FindDirectionReference(source, secondSourceOffset, stride, coefficientShift, out int secondExpectedVariance); + Av1CdefFilter.FindDirections( + source, + sourceOffset, + secondSourceOffset, + stride, + coefficientShift, + out int firstActualDirection, + out int firstActualVariance, + out int secondActualDirection, + out int secondActualVariance); + + Assert.Equal(expectedDirection, firstActualDirection); + Assert.Equal(expectedVariance, firstActualVariance); + Assert.Equal(secondExpectedDirection, secondActualDirection); + Assert.Equal(secondExpectedVariance, secondActualVariance); + } + } + } + + /// + /// Populates two adjacent 8x8 blocks with deterministic directional samples. + /// + /// The destination source plane. + /// The first populated sample. + /// The source row stride. + /// The deterministic pattern index. + /// The sample precision. + private static void PopulateDirectionSource(Span source, int sourceOffset, int sourceStride, int pattern, int bitDepth) + { + int coefficientShift = bitDepth - 8; + int maximum = (1 << bitDepth) - 1; + for (int row = 0; row < 8; row++) + { + for (int column = 0; column < 16; column++) + { + int localColumn = column & 7; + int direction = column < 8 ? pattern : (pattern + 4) & 7; + int line = GetDirectionLineReference(direction, row, localColumn); + + // Samples are constant along the requested geometric line and vary between lines. Direction search + // therefore minimizes reconstruction error in that direction while still exercising nonuniform values. + int value = (24 + (line * 14)) << coefficientShift; + source[sourceOffset + (row * sourceStride) + column] = (ushort)(value & maximum); + } + } + } + + /// + /// Maps one source coordinate to its line in an AV1 direction independently of the production implementation. + /// + /// The zero-based AV1 direction index. + /// The source row. + /// The source column. + /// The zero-based line index. + private static int GetDirectionLineReference(int direction, int row, int column) => direction switch + { + 0 => row + column, + 1 => row + (column / 2), + 2 => row, + 3 => 3 + row - (column / 2), + 4 => 7 + row - column, + 5 => 3 - (row / 2) + column, + 6 => column, + _ => (row / 2) + column + }; + + /// + /// Exercises all normative block dimensions, directions, strength combinations, output types, and coded precisions. + /// + private static void ValidateFilters() + { + (int Width, int Height)[] dimensions = [(4, 4), (4, 8), (8, 4), (8, 8)]; + foreach (int bitDepth in new[] { 8, 10, 12 }) + { + int coefficientShift = bitDepth - 8; + int scale = 1 << coefficientShift; + ushort[] source = CreateBorderedSource(bitDepth); + int sourceOffset = (SourceBorder * SourceStride) + SourceBorder; + (int Primary, int Secondary)[] strengths = [(0, 0), (4 * scale, 0), (0, 2 * scale), (5 * scale, 2 * scale)]; + + foreach ((int blockWidth, int blockHeight) in dimensions) + { + foreach (int direction in Enumerable.Range(0, 8)) + { + foreach ((int primaryStrength, int secondaryStrength) in strengths) + { + if (bitDepth == 8) + { + AssertByteFilter( + source, + sourceOffset, + blockWidth, + blockHeight, + primaryStrength, + secondaryStrength, + direction, + coefficientShift); + } + + AssertUInt16Filter( + source, + sourceOffset, + blockWidth, + blockHeight, + primaryStrength, + secondaryStrength, + direction, + coefficientShift); + } + } + } + } + } + + /// + /// Creates a deterministic image whose top-left output block touches the unavailable-neighbor border. + /// + /// The source sample precision. + /// The bordered 16-bit source plane. + private static ushort[] CreateBorderedSource(int bitDepth) + { + ushort[] source = Enumerable.Repeat(Av1CdefFilter.VeryLarge, SourceStride * SourceStride).ToArray(); + int maximum = (1 << bitDepth) - 1; + int scale = 1 << (bitDepth - 8); + for (int row = SourceBorder; row < SourceStride - SourceBorder; row++) + { + for (int column = SourceBorder; column < SourceStride - SourceBorder; column++) + { + int localRow = row - SourceBorder; + int localColumn = column - SourceBorder; + int value = (72 + (localRow * 9) + (localColumn * 5) + ((localRow * localColumn) & 15)) * scale; + source[(row * SourceStride) + column] = (ushort)Math.Min(value, maximum); + } + } + + return source; + } + + /// + /// Verifies one eight-bit output block while retaining untouched destination padding in the comparison. + /// + /// The bordered source plane. + /// The first source sample in the block. + /// The output block width. + /// The output block height. + /// The primary filter strength. + /// The secondary filter strength. + /// The primary filter direction. + /// The source precision shift. + private static void AssertByteFilter( + ushort[] source, + int sourceOffset, + int blockWidth, + int blockHeight, + int primaryStrength, + int secondaryStrength, + int direction, + int coefficientShift) + { + const int destinationStride = 12; + const int destinationOffset = destinationStride + 1; + byte[] expected = Enumerable.Repeat((byte)231, destinationStride * 10).ToArray(); + byte[] actual = (byte[])expected.Clone(); + int damping = 5 + coefficientShift; + + FilterReference( + source, + sourceOffset, + SourceStride, + expected, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + damping, + coefficientShift, + blockWidth, + blockHeight); + + Av1CdefFilter.FilterBlock( + source, + sourceOffset, + SourceStride, + actual, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + damping, + damping, + coefficientShift, + blockWidth, + blockHeight); + + Assert.Equal(expected, actual); + } + + /// + /// Verifies one 16-bit output block while retaining untouched destination padding in the comparison. + /// + /// The bordered source plane. + /// The first source sample in the block. + /// The output block width. + /// The output block height. + /// The primary filter strength. + /// The secondary filter strength. + /// The primary filter direction. + /// The source precision shift. + private static void AssertUInt16Filter( + ushort[] source, + int sourceOffset, + int blockWidth, + int blockHeight, + int primaryStrength, + int secondaryStrength, + int direction, + int coefficientShift) + { + const int destinationStride = 12; + const int destinationOffset = destinationStride + 1; + ushort[] expected = Enumerable.Repeat((ushort)60000, destinationStride * 10).ToArray(); + ushort[] actual = (ushort[])expected.Clone(); + int damping = 5 + coefficientShift; + + FilterReference( + source, + sourceOffset, + SourceStride, + expected, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + damping, + coefficientShift, + blockWidth, + blockHeight); + + Av1CdefFilter.FilterBlock( + source, + sourceOffset, + SourceStride, + actual, + destinationOffset, + destinationStride, + primaryStrength, + secondaryStrength, + direction, + damping, + damping, + coefficientShift, + blockWidth, + blockHeight); + + Assert.Equal(expected, actual); + } + + /// + /// Finds one direction and variance using the scalar AV1 definition independently of the production layouts. + /// + /// The source plane. + /// The first sample in the 8x8 block. + /// The source row stride. + /// The source precision shift. + /// Receives the directional variance. + /// The selected direction. + private static int FindDirectionReference( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + int coefficientShift, + out int variance) + { + int[,] partials = new int[8, 15]; + int[] costs = new int[8]; + int[] divisions = [0, 840, 420, 280, 210, 168, 140, 120, 105]; + for (int row = 0; row < 8; row++) + { + for (int column = 0; column < 8; column++) + { + int value = (source[sourceOffset + (row * sourceStride) + column] >> coefficientShift) - 128; + partials[0, row + column] += value; + partials[1, row + (column / 2)] += value; + partials[2, row] += value; + partials[3, 3 + row - (column / 2)] += value; + partials[4, 7 + row - column] += value; + partials[5, 3 - (row / 2) + column] += value; + partials[6, column] += value; + partials[7, (row / 2) + column] += value; + } + } + + for (int line = 0; line < 8; line++) + { + costs[2] += partials[2, line] * partials[2, line]; + costs[6] += partials[6, line] * partials[6, line]; + } + + costs[2] *= divisions[8]; + costs[6] *= divisions[8]; + for (int line = 0; line < 7; line++) + { + costs[0] += ((partials[0, line] * partials[0, line]) + (partials[0, 14 - line] * partials[0, 14 - line])) * divisions[line + 1]; + costs[4] += ((partials[4, line] * partials[4, line]) + (partials[4, 14 - line] * partials[4, 14 - line])) * divisions[line + 1]; + } + + costs[0] += partials[0, 7] * partials[0, 7] * divisions[8]; + costs[4] += partials[4, 7] * partials[4, 7] * divisions[8]; + for (int direction = 1; direction < 8; direction += 2) + { + for (int line = 0; line < 5; line++) + { + costs[direction] += partials[direction, 3 + line] * partials[direction, 3 + line]; + } + + costs[direction] *= divisions[8]; + for (int line = 0; line < 3; line++) + { + costs[direction] += ((partials[direction, line] * partials[direction, line]) + + (partials[direction, 10 - line] * partials[direction, 10 - line])) * divisions[(2 * line) + 2]; + } + } + + int bestCost = 0; + int bestDirection = 0; + for (int direction = 0; direction < 8; direction++) + { + if (costs[direction] > bestCost) + { + bestCost = costs[direction]; + bestDirection = direction; + } + } + + variance = (bestCost - costs[(bestDirection + 4) & 7]) >> 10; + return bestDirection; + } + + /// + /// Applies the independent scalar filter definition to eight-bit output storage. + /// + private static void FilterReference( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + Span destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int damping, + int coefficientShift, + int blockWidth, + int blockHeight) + { + for (int row = 0; row < blockHeight; row++) + { + for (int column = 0; column < blockWidth; column++) + { + destination[destinationOffset + (row * destinationStride) + column] = (byte)FilterSampleReference( + source, + sourceOffset + (row * sourceStride) + column, + sourceStride, + primaryStrength, + secondaryStrength, + direction, + damping, + coefficientShift); + } + } + } + + /// + /// Applies the independent scalar filter definition to 16-bit output storage. + /// + private static void FilterReference( + ReadOnlySpan source, + int sourceOffset, + int sourceStride, + Span destination, + int destinationOffset, + int destinationStride, + int primaryStrength, + int secondaryStrength, + int direction, + int damping, + int coefficientShift, + int blockWidth, + int blockHeight) + { + for (int row = 0; row < blockHeight; row++) + { + for (int column = 0; column < blockWidth; column++) + { + destination[destinationOffset + (row * destinationStride) + column] = (ushort)FilterSampleReference( + source, + sourceOffset + (row * sourceStride) + column, + sourceStride, + primaryStrength, + secondaryStrength, + direction, + damping, + coefficientShift); + } + } + } + + /// + /// Computes one independently filtered sample from its primary and secondary neighbors. + /// + private static int FilterSampleReference( + ReadOnlySpan source, + int sourceIndex, + int sourceStride, + int primaryStrength, + int secondaryStrength, + int direction, + int damping, + int coefficientShift) + { + bool enablePrimary = primaryStrength != 0; + bool enableSecondary = secondaryStrength != 0; + bool clippingRequired = enablePrimary && enableSecondary; + int primaryTapSet = (primaryStrength >> coefficientShift) & 1; + int sample = source[sourceIndex]; + int sum = 0; + int minimum = sample; + int maximum = sample; + + for (int tap = 0; tap < 2; tap++) + { + if (enablePrimary) + { + int offset = GetDirectionOffsetReference(direction, tap, sourceStride); + int neighbor0 = source[sourceIndex + offset]; + int neighbor1 = source[sourceIndex - offset]; + int weight = primaryTapSet == 0 ? (tap == 0 ? 4 : 2) : 3; + sum += weight * ConstrainReference(neighbor0 - sample, primaryStrength, damping); + sum += weight * ConstrainReference(neighbor1 - sample, primaryStrength, damping); + + if (clippingRequired) + { + maximum = neighbor0 != Av1CdefFilter.VeryLarge ? Math.Max(maximum, neighbor0) : maximum; + maximum = neighbor1 != Av1CdefFilter.VeryLarge ? Math.Max(maximum, neighbor1) : maximum; + minimum = Math.Min(minimum, Math.Min(neighbor0, neighbor1)); + } + } + + if (enableSecondary) + { + int offset0 = GetDirectionOffsetReference((direction + 2) & 7, tap, sourceStride); + int offset1 = GetDirectionOffsetReference((direction + 6) & 7, tap, sourceStride); + int neighbor0 = source[sourceIndex + offset0]; + int neighbor1 = source[sourceIndex - offset0]; + int neighbor2 = source[sourceIndex + offset1]; + int neighbor3 = source[sourceIndex - offset1]; + int weight = tap == 0 ? 2 : 1; + sum += weight * ConstrainReference(neighbor0 - sample, secondaryStrength, damping); + sum += weight * ConstrainReference(neighbor1 - sample, secondaryStrength, damping); + sum += weight * ConstrainReference(neighbor2 - sample, secondaryStrength, damping); + sum += weight * ConstrainReference(neighbor3 - sample, secondaryStrength, damping); + + if (clippingRequired) + { + maximum = neighbor0 != Av1CdefFilter.VeryLarge ? Math.Max(maximum, neighbor0) : maximum; + maximum = neighbor1 != Av1CdefFilter.VeryLarge ? Math.Max(maximum, neighbor1) : maximum; + maximum = neighbor2 != Av1CdefFilter.VeryLarge ? Math.Max(maximum, neighbor2) : maximum; + maximum = neighbor3 != Av1CdefFilter.VeryLarge ? Math.Max(maximum, neighbor3) : maximum; + minimum = Math.Min(minimum, Math.Min(Math.Min(neighbor0, neighbor1), Math.Min(neighbor2, neighbor3))); + } + } + } + + int filtered = sample + ((8 + sum - (sum < 0 ? 1 : 0)) >> 4); + return clippingRequired ? Math.Clamp(filtered, minimum, maximum) : filtered; + } + + /// + /// Applies the scalar AV1 constrain equation independently of the production implementation. + /// + private static int ConstrainReference(int difference, int threshold, int damping) + { + int shift = Math.Max(0, damping - BitOperations.Log2((uint)threshold)); + int magnitude = Math.Abs(difference); + int constrained = Math.Clamp(threshold - (magnitude >> shift), 0, magnitude); + return difference < 0 ? -constrained : constrained; + } + + /// + /// Converts a direction and tap to a signed source offset independently of the production implementation. + /// + private static int GetDirectionOffsetReference(int direction, int tap, int stride) + { + int[] x0 = [1, 1, 1, 1, 1, 0, 0, 0]; + int[] y0 = [-1, 0, 0, 0, 1, 1, 1, 1]; + int[] x1 = [2, 2, 2, 2, 2, 1, 0, -1]; + int[] y1 = [-2, -1, 0, 1, 2, 2, 2, 2]; + return tap == 0 ? (y0[direction] * stride) + x0[direction] : (y1[direction] * stride) + x1[direction]; + } +}