diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 2c5450e0e..662ff1e21 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -29,7 +29,7 @@ Checkboxes may be marked complete only when the implementation and the verificat ## Delivery dashboard -Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `32d5e3b51`, including the completed AV1 transform, OBU-framing, intra-block-copy, 12-profile reconstruction, and layered-item property checkpoints. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. +Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `f6d3da2b31`, including the completed AV1 transform, OBU-framing, intra-block-copy, 12-profile reconstruction, layered-item property, and layered reference/header-state checkpoints. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. Status meanings: @@ -40,13 +40,13 @@ Status meanings: Current development stage: **Stage 3 — complete AV1 still-image decoding.** The transform checkpoint is closed: forward transforms use one libaom-shaped SIMD-first operator architecture across `Vector512`, `Vector256`, and `Vector128`, with scalar fallback; inverse production traversal uses the verified `Vector256` and `Vector128` tiers with scalar fallback; and implementation-mechanic type and file suffixes have been removed. Neither AV1 nor HEVC production encoding is implemented. -Immediate checkpoint: **complete layered AV1 image-item decoding through the existing image-only container surface.** This includes `a1op`, `lsel`, and `a1lx` properties, operating-point selection, dependency-preserving layer consumption, and final or explicitly selected spatial-layer output for color, alpha, and grid items. It requires stateful AV1 reference/CDF reconstruction; it must not be represented as animation or expanded into a general ISO BMFF/video model. +Immediate checkpoint: **complete layered AV1 image-item decoding through the existing image-only container surface.** This includes `a1op`, `lsel`, and `a1lx` properties, operating-point selection, dependency-preserving layer consumption, and final or explicitly selected spatial-layer output for color, alpha, and grid items. The bounded decoder session now retains reference owners and the header, entropy, segmentation, loop-filter, global-motion, and temporal motion-field state required by dependent layers. Inter tile syntax and reconstruction still need to consume that state. This work must not be represented as animation or expanded into a general ISO BMFF/video model. | Order | Delivery stage | State | Delivered state | Gate that remains open | | --- | --- | --- | --- | --- | | 1 | Baseline, provenance, documentation, and public contract | In progress | Pinned codec references, a bounded image-only scope, encoder options, typed bit depth, decoder-option propagation, and extensive HEIF documentation exist. | Complete the all-file documentation audit, record a fresh Release baseline, finish distinct public HEIC/AVIF save boundaries, and close API review. | | 2 | Bounded HEIF item and image-sequence container | In progress | Still-item parsing, grids, auxiliary alpha, metadata properties, bounded image-sequence tracks, Identify, and all-sync AV1 sequence presentation are connected. | Complete adversarial boundary coverage, remaining item/property behavior, reference-dependent sequence reconstruction, and the bounded sequence writer. | -| 3 | Still-image AV1 and HEVC decoding | **Current** | HEVC reconstruction reaches exact HM/libheif fixtures across the recorded 8/10/12-bit and chroma cases. AV1 includes bounded OBU framing, reconstruction, filters, grain, color, transforms, intra-block copy, and an exact independent 12-profile bit-depth/chroma matrix through every dispatch tier. | Remove every remaining valid AV1 still-image unsupported branch with independent compression-tool vectors, then complete the remaining HEVC profile and Range Extensions matrix. | +| 3 | Still-image AV1 and HEVC decoding | **Current** | HEVC reconstruction reaches exact HM/libheif fixtures across the recorded 8/10/12-bit and chroma cases. AV1 includes bounded OBU framing, reconstruction, filters, grain, color, transforms, intra-block copy, an exact independent 12-profile bit-depth/chroma matrix through every dispatch tier, and retained layered reference/header/CDF state. | Complete inter tile syntax and reconstruction, including temporal segmentation consumption and motion/global/warped prediction, remove every other valid AV1 still-image unsupported branch with independent compression-tool vectors, then complete the remaining HEVC profile and Range Extensions matrix. | | 4 | Complete decoded presentation and animation | In progress | Shared SIMD-first AV1/HEVC color conversion, ICC application, grids, transforms, direct planar alpha composition, frame metadata, repetition, and independently decodable AV1 sequence samples exist. | Close the full color/ICC cross-product, HEVC sequence decoding, AV1/HEVC reference-dependent samples, frame-local metadata/alpha behavior, and independent animated decode vectors. | | 5 | AV1/AVIF encoding | Not started | RGB-to-planar conversion, forward transforms, OBU writer foundations, options, and container-writing infrastructure exist. | `HeifEncoderCore` still rejects AV1. Implement a real independently decodable lossy/lossless AV1 payload and the complete AVIF item/metadata matrix. | | 6 | HEVC/HEIC encoding | Not started | Shared input color conversion, options, and HEIF writer infrastructure exist. | `HeifEncoderCore` still rejects HEVC. Implement a real independently decodable lossy/lossless HEVC payload and the complete HEIC item/metadata matrix. | @@ -63,10 +63,16 @@ Immediate checkpoint: **complete layered AV1 image-item decoding through the exi - [x] Validate `a1lx` layer boundaries against the logical item size and restrict concrete `lsel` decoding to the cumulative payload through the selected spatial layer without copying item bytes. - [x] Apply the selected `a1op` operating-point mask while consuming extended OBUs and validate the selected index against the parsed sequence header. - [x] Store the eight fixed reference-validity, order-hint, and map-index tables inline on the frame header, retaining complete multi-bit order hints without per-header array allocations. - - [ ] Preserve reconstruction, reference-frame, primary-CDF, segmentation, loop-filter, and motion state across every dependent layer in one image-item decoder session. + - [x] Enforce the bounded frame lifecycle across temporal delimiters, frame headers, tile coverage, and payload completion, publishing parser reference state only after successful decoder completion. The real two-layer libavif-derived lifecycle fixture verifies frame completion, retained-slot occupancy, reference resolution, and inherited dimensions through a fake tile lifecycle; it does not verify inter reconstruction or pixels. + - [x] Retain each successfully reconstructed reference as the sole owner of its ungrained sample planes, frame header, frame information, and published entropy snapshot; refresh only the selected slots, extend reference borders once after in-loop filtering, preserve a separate grained presentation buffer when required, and reset the store at an accepted sequence boundary. + - [x] Parse full and short reference signaling against retained-slot occupancy separately from frame-ID validity, resolve the primary-reference slot, implement `frame_size_with_refs`, and read high-precision-motion-vector, interpolation-filter, switchable-motion-mode, and reference-frame-motion-vector flags. + - [x] Initialize each frame's working CDF graph from the resolved primary snapshot, inherit segmentation feature data and an unchanged contiguous segmentation map, inherit loop-filter delta state, parse and inherit global-motion parameters, and initialize/project the retained per-8x8 temporal motion field. + - [x] Match libaom's bounded range-decoder consumed-bit accounting and tile trailing-bit validation so implicit zero refill cannot conceal truncated entropy data. Validate before publishing the selected CDF, reset decoder state after failure, and unwind every successful frame/block workspace rent when a later constructor allocation fails. A real truncated palette tile, decoder reuse, parser lifecycle, and allocator-identity tests cover these boundaries. + - [ ] Consume the retained segmentation map through `seg_id_predicted` when temporal segmentation is enabled. The current intra-only tile path does not read that inter-block decision. - [ ] Implement the complete inter-frame entropy, mode, motion-vector, compound-prediction, inter-prediction, and warped/global-motion paths permitted by the image profile. - [x] Implement allocation-free SIMD-first translational single-reference interpolation for regular, smooth, sharp, and bilinear filters across 8/10/12-bit samples. The predictor mirrors JPEG's closed static operator architecture, descends through `Vector512`, `Vector256`, and `Vector128` before scalar fallback, and passes the exact independent convolution oracle through `FeatureTestRunner`. - - [ ] Connect reference selection, motion-vector derivation, compound modes, warped/global motion, and reconstructed-plane writes through tile decoding, then verify them with independently encoded inter-frame AV1 image-layer fixtures. + - [ ] Decode inter-block reference selection, motion-vector derivation, compound and inter-intra modes, masked blending, OBMC, and warped/global-motion prediction, then write predicted and residual samples through tile reconstruction. + - [ ] Verify every connected mode and filter with independently encoded dependent-layer AV1 image-item fixtures and exact native-plane comparisons. - [ ] Return the explicitly selected spatial layer or the final displayed layer, keeping reference reconstruction separate from display-only film grain. - [ ] Verify color and auxiliary-alpha output exactly against both pinned libavif progressive fixtures under normal SIMD dispatch and all required `FeatureTestRunner` fallbacks. - [ ] Correct the audited 12-bit inverse ADST4, Identity4, and Identity16 SIMD arithmetic by widening only the libaom-widened multiply/accumulate operations, with exact conformant-range vectors and `FeatureTestRunner` coverage. @@ -289,7 +295,7 @@ This snapshot pins or classifies the available references and failures; it does | `Av1WienerFilter` | AV1 sections 7.17.4 and 7.17.5 Wiener restoration filtering and coefficient derivation | libaom `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/convolve.c`, and `av1/common/convolve.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the implicit center-sample contribution, separable horizontal/vertical rounding, bit-depth-dependent 16-bit intermediate range, and final 8/10/12-bit clipping. Reuse `Vector128_.MultiplyAddAdjacent` for the contiguous horizontal eight-tap product with an exact scalar fallback. Keep the restoration stage disabled until stripe boundaries and self-guided filtering are both complete. | | `Av1SelfGuidedFilter` | AV1 sections 7.17.2 and 7.17.3 self-guided and box-filter processes | libaom `av1/common/restoration.c` and `av1/common/restoration.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design the window layout, caller-owned scratch, local statistics, and projection traversal from libaom's architecture-specific kernels so the production path is SIMD-first. Implement the equivalent scalar fallback through the same contract. Preserve the sixteen normative radius/variance parameter sets, local mean and variance normalization, alternating-row radius-two optimization, decoded projection-coefficient behavior, signed rounding, and 8/10/12-bit clipping. Keep this image-reconstruction stage disabled until restoration stripe boundaries are complete. | | `Av1LoopRestorationBoundary`, `Av1LoopRestorationDecoder`, and `Av1FrameDecoder` restoration-stage ordering | AV1 section 7.17 loop restoration, including striped boundary semantics | libaom `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/resize.c`, and `av1/decoder/decodeframe.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve two deblocked rows at internal 64-luma stripe boundaries before CDEF, apply the existing normative SIMD-backed super-resolution kernel to saved rows when scaled, use post-CDEF/super-resolution samples at frame edges, extend the final restoration unit up to 150 percent of nominal size, and filter from immutable plane snapshots into separate output planes. This is bounded still-image reconstruction state, not retained reference-frame, track, timing, or playback state. | -| `Av1FilmGrainDecoder`, `Av1FilmGrainGaussianSequence`, `Av1FilmGrainNoise`, `Av1FilmGrainOverlap`, and `Av1FilmGrainSampleOperator` | AV1 section 7.18 film-grain synthesis | libaom `av1/decoder/grain_synthesis.c`, `av1/decoder/grain_synthesis.h`, and `aom_dsp/grain_params.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative 2,048-sample Gaussian sequence as compile-time span data, the serial linear-feedback shift register and luma/chroma autoregressive templates, scaling lookup interpolation, 32x32 block selection, boundary overlap, restricted-range clipping, monochrome and 4:2:0/4:2:2/4:4:4 layouts, and 8/10/12-bit arithmetic. Use allocator-owned scratch. Apply scaling and noise through AVX2 gather arithmetic at every bit depth, retain the measured cross-platform 128-bit high-bit-depth path, and use the exact scalar path for 8-bit machines without AVX2 and for vector tails. Process horizontal overlap through preferred-native 512-bit, then 256-bit, 128-bit, and scalar tiers; retain scalar vertical overlap because each output row exposes only one or two strided samples. Keep SIMD width, ISA, sample storage, and bit depth out of folders, namespaces, files, and type names. Apply grain only to displayed still-image samples after all in-loop filters; reference-frame parameter inheritance remains sequence-playback state and is outside this codec scope. | +| `Av1FilmGrainDecoder`, `Av1FilmGrainGaussianSequence`, `Av1FilmGrainNoise`, `Av1FilmGrainOverlap`, and `Av1FilmGrainSampleOperator` | AV1 section 7.18 film-grain synthesis | libaom `av1/decoder/grain_synthesis.c`, `av1/decoder/grain_synthesis.h`, and `aom_dsp/grain_params.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative 2,048-sample Gaussian sequence as compile-time span data, the serial linear-feedback shift register and luma/chroma autoregressive templates, scaling lookup interpolation, 32x32 block selection, boundary overlap, restricted-range clipping, monochrome and 4:2:0/4:2:2/4:4:4 layouts, and 8/10/12-bit arithmetic. Use allocator-owned scratch. Apply scaling and noise through AVX2 gather arithmetic at every bit depth, retain the measured cross-platform 128-bit high-bit-depth path, and use the exact scalar path for 8-bit machines without AVX2 and for vector tails. Process horizontal overlap through preferred-native 512-bit, then 256-bit, 128-bit, and scalar tiers; retain scalar vertical overlap because each output row exposes only one or two strided samples. Keep SIMD width, ISA, sample storage, and bit depth out of folders, namespaces, files, and type names. Apply grain only to displayed samples after all in-loop filters, inherit reference-selected parameters through the bounded image-layer decoder state, and keep the ungrained reconstruction in every refreshed reference slot. This does not add sequence playback or video scope. | | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. | | `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. | | `Av1ForwardTransformer`, `Av1Inverse2dTransformer`, `Av1Transform2dFlipConfiguration`, the forward/inverse 1-D operator structs, and `Av1Transform1dMath` | AV1 forward transform definitions and section 7.11.2 inverse transform and reconstruction | libaom `av1/encoder/av1_fwd_txfm2d_hwy.h`, its AVX-512 and AVX2 instantiations, `av1/encoder/av1_fwd_txfm1d.c`, `av1/common/av1_inv_txfm1d_cfg.h`, `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, the x86 AVX2/SSE4 implementations, and the corresponding Neon implementations at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative staged DCT, ADST, and identity arithmetic, direction-specific stage ranges and shifts, transposition, clipping, and high-bit-depth sample addition. Stateless static-generic operators follow ImageSharp's JPEG color-transform pattern. Forward operators share one behavior model across scalar, `Vector128`, `Vector256`, and `Vector512`, selecting packed or expanded lane storage at the 2-D boundary. Inverse production traversal retains the verified scalar, `Vector128`, and `Vector256` tiers until a wider upstream shape and complete-block benefit are both established. | @@ -364,7 +370,7 @@ This assessment was reconciled with the source tree on 2026-08-26. Unless a resu ### AV1 decoder -- The single-still `Av1Decoder` path now parses tile state before allocating and reconstructing one independently decodable frame, and it disposes the reconstruction planes after pixel conversion. It deliberately does not retain animation/video reference frames or implement `show_existing_frame` playback state. +- The bounded `Av1Decoder` session parses all tile state before allocating and reconstructing each coded image layer. After successful completion it retains the ungrained reference planes, frame header, frame information, and published entropy snapshot in the refreshed slots, while presentation-only ownership remains separate. A new accepted sequence header resets both parser and retained-owner state. This is dependency reconstruction within one bounded image item; `show_existing_frame` playback remains rejected and no animation/video reference model is exposed. - Transform coefficient entropy derivation and updates now address the above contexts relative to the tile column and the left contexts relative to the current superblock row, preserve luma coordinates independently of chroma subsampling, and test every packed context entry for the libaom any-nonzero rule. Extended vertical partition updates advance the mode-information column rather than the row. The existing multi-superblock 4:4:4 AVIF fixture now completes tile parsing; independent coefficient-context vectors across tile boundaries, chroma layouts, bit depths, and edge-clipped transforms remain required. - The reconstruction pipeline now records plane-relative transform geometry, preserves tile-local delta-Q and delta-LF predictors, derives segmentation and reference-adjusted filter levels, and runs the exact AV1 4-, 6-, 8-, and 14-tap deblocking kernels in normative vertical-then-horizontal order. Deblocking uses the same closed edge-operator architecture as the HEVC filter, with operators specialized by sample storage and orientation, `Vector128` lanes representing the four rows or columns along an edge, and an allocation-free scalar fallback for disabled intrinsics. Exact native-plane comparison with pinned scalar libaom output now verifies active deblocking and complete reconstruction for real 8-bit 4:2:0, 10-bit 4:4:4, and 12-bit 4:4:4 content; genuine AVIF containers separately verify presentation and public bit-depth metadata. The pipeline then applies CDEF through one semantic filter architecture: paired AVX2 and single-block `Vector128` direction analysis, closed primary/secondary strength operators, packed 4x4/4x8/8x4/8x8 constrained filtering, byte/16-bit output operators, and an exact allocation-free scalar fallback. Decoder orchestration now owns the immutable plane snapshots and clean direction/variance maps through ImageSharp's memory allocator, widens 8-bit source rows with the same AVX2/128-bit/scalar tiers as libaom, lists each unit's non-skipped blocks in fixed inline storage, analyzes listed blocks in pairs, and writes filtered bytes or 16-bit samples directly to the frame planes. Independently encoded active-CDEF 8-bit 4:2:0 and 10/12-bit 4:4:4 streams match every visible native sample produced by pinned scalar libaom with restoration disabled. Independently encoded AVIF containers at the same three bit depths also match pinned scalar-libavif presentation exactly under normal, 256-bit, 128-bit, and scalar color-conversion dispatch. Active super-resolution derives the Appendix A bounded coded width and applies the exact 64-phase, 8-tap horizontal filter with aligned reconstruction-edge input, 8/10/12-bit clipping, and the existing cross-platform `Vector128_.MultiplyAddAdjacent` helper. Independently encoded active-super-resolution AV1 streams at 8, 10, and 12 bits match every visible native sample produced by pinned scalar libaom under normal and forced-scalar dispatch. Independently packaged AVIF containers at the same bit depths retain matching libavif profile, dimensions, chroma, and CICP properties, require active super-resolution in their actual AV1 item, and match pinned scalar-libavif presentation exactly under normal, 256-bit, 128-bit, and scalar color-conversion dispatch. Loop restoration follows super-resolution, preserves the required pre-CDEF deblocked context at internal stripes, and applies decoded Wiener or self-guided units from immutable plane snapshots. Independently encoded active-restoration streams at 8, 10, and 12 bits now match every native sample from pinned scalar libaom across AVX2, 128-bit, and scalar dispatch, with the fixture matrix proving both Wiener and self-guided unit selection. The matching AVIF containers also match pinned scalar-libavif presentation exactly, and an independent direct-window definition verifies all sixteen self-guided parameter sets at each supported bit depth across vector and scalar dispatch. Combined 8-bit 4:2:0, 10-bit 4:2:2, and 12-bit 4:4:4 streams additionally verify restoration-unit boundaries after super-resolution, including clipped chroma transform traversal at a coded-frame edge. - The visible still-image path applies the complete self-contained film-grain parameter set after all in-loop filters. Independently encoded pinned-libaom vectors match every native sample at 8, 10, and 12 bits across monochrome, 4:2:0, 4:2:2, and 4:4:4 layouts, full and restricted ranges, identity-matrix signaling, overlap, and odd 33x11 frame extension. `FeatureTestRunner` verifies normal, AVX-disabled, and fully scalar dispatch. A full-HD-equivalent 4:2:0 benchmark reports zero allocation: 8-bit AVX2 is 2.335 ms versus 5.806 ms scalar, while 12-bit AVX2 is 3.195 ms, cross-platform 128-bit is 7.382 ms, and scalar is 8.614 ms on the measured Ryzen platform. The slower 8-bit 128-bit lookup construction is deliberately not dispatched. @@ -393,7 +399,7 @@ This assessment was reconciled with the source tree on 2026-08-26. Unless a resu ### Tests - HEVC coverage includes exact native-plane comparison with HM output, exact complete-image comparison with pinned libheif/libde265 references, and the 10 official Sony GENERAL Range Extensions first-picture fixtures across 8/10/12-bit monochrome, 4:2:0, 4:2:2, and 4:4:4 reconstruction. The remaining exposed profiles and individual Range Extensions tools still need exact independent vectors. -- AV1 has focused bitstream, prediction, entropy, reconstruction, filter, film-grain, color, and transform coverage, plus real libavif inputs. Valid still-image syntax paths still contain explicit unsupported branches, so the independent AV1 decode matrix is not complete. +- AV1 has focused bitstream, prediction, entropy, reconstruction, filter, film-grain, color, and transform coverage, plus real libavif inputs. A real two-layer libavif-derived OBU stream verifies the bounded frame lifecycle, retained-slot occupancy, resolved inter references, and `frame_size_with_refs` dimensions through a fake tile lifecycle. A real palette stream truncated inside its tile entropy payload verifies libaom-equivalent overflow/trailing-bit rejection and decoder-session recovery. The current Release checkpoint passes all 2,422 selected entropy, ownership, reference, predictor, intra-block-copy, and transform cases. It does not decode inter tile syntax or compare dependent-layer reconstructed pixels. Valid still-image syntax paths still contain explicit unsupported branches, so the independent AV1 decode matrix is not complete. - The AV1 transform matrix verifies scalar, `Vector128`, `Vector256`, and `Vector512` forward representations plus the production inverse tiers across every valid size/type combination and supported bit depth. All 511 focused forward and inverse cases pass in Release; `FeatureTestRunner` isolates hardware tiers, every two-dimensional configuration exercises production dispatch, and the complete-block benchmark records zero managed allocation. - Independent libavif fixtures cover primary, grid, auxiliary-alpha, ICC, metadata-skipping, and all-sync image-sequence presentation. Reference-dependent AV1 and HEVC sequence reconstruction and independent HEVC ICC sequence coverage remain open. - Focused decoder-option tests cover strict, ancillary-only, image-data, and metadata-skipping behavior for still items and sequence samples. Complete adversarial dimension, allocation, malformed-container, and resource-limit coverage remains open. @@ -511,7 +517,7 @@ Implement and verify in dependency order: - [ ] OBU framing, sequence headers, frame headers, tile groups, byte alignment, and trailing bits. - [x] Isolate every declared OBU payload with a zero-copy bounded span reader, advance ignored metadata and reserved units without parsing their bytes as headers, validate padding and trailing bytes, enforce primary/redundant/combined frame-header order, accept the final unsized low-overhead OBU by consuming the bounded image-item remainder, and reject lengths that cross the containing boundary. Focused malformed-input tests and independent 8/10/12-bit AVIF reconstruction pass in Release. -- [ ] One coherent decoder lifecycle that retains parsed frame and tile state and disposes all buffers deterministically. +- [x] One coherent bounded decoder lifecycle retains parsed tile state until successful frame completion, commits frame-owned reference samples and metadata only after reconstruction, resets retained state at a new sequence boundary, and disposes tile, reference, entropy, and presentation ownership deterministically. The real two-layer lifecycle fixture proves the header/reference ownership boundary through a fake tile reader; dependent inter reconstruction remains covered by the unchecked items below. - [ ] Tile partitioning, mode information, segmentation, delta quantization, transform-size selection, coefficient token decode, inverse quantization, and inverse transforms. - [x] Match libaom's depth-first traversal and frame-edge behavior for all ten AV1 partition types. Independent 8/10/12-bit streams collectively select every terminal partition shape and contain nested block geometry that requires recursive `Split` traversal; their complete native planes remain byte-exact under normal hardware dispatch and the scalar fallback. - [x] Verify the complete 8/10/12-bit monochrome, 4:2:0, 4:2:2, and 4:4:4 base profile matrix against pinned libaom native planes and pinned libavif presentation output. All 12 fixtures match exactly with normal dispatch, AVX-512 disabled, AVX disabled, and all hardware intrinsics disabled through `FeatureTestRunner`. @@ -524,8 +530,11 @@ Implement and verify in dependency order: - [x] Decode tile-adaptive integer displacement vectors, derive and validate spatial references, apply the inter transform sets, and reconstruct luma and subsampled chroma through allocation-free `Vector512`/`Vector256`/`Vector128` operators with exact-width stores and scalar fallback. `FeatureTestRunner` verifies every transform width, bit-depth storage path, chroma phase, intrinsic tier, scalar continuation, and destination-padding boundary. - [x] Verify displacement-vector entropy, spatial candidate ordering, wavefront legality, native 8/10/12-bit reconstruction, and presented output against independently encoded opaque intra-block-copy AVIF fixtures from the pinned generic libaom/libavif reference. The fixtures require actual intra-block-copy block selection, compare every native plane sample from retained scalar-decoder Y4M output, and compare every presented RGBA byte exactly under normal hardware dispatch and each narrower fallback configuration without a tolerance. - [ ] Inter-frame prediction for layered still-image items. + - [x] Retain completed ungrained reference planes, frame headers, frame information, and published CDF snapshots in one eight-slot owner; apply refresh flags only after successful completion; resolve full and short reference signaling against occupancy and frame-ID validity; and implement primary-reference selection and `frame_size_with_refs`. + - [x] Inherit primary-reference CDFs, segmentation features and unchanged maps, loop-filter deltas, and same-role global-motion parameters. Initialize the frame-owned per-8x8 temporal motion field and project eligible retained motion vectors in normative reference order. - [x] Implement allocation-free SIMD-first translational single-reference interpolation for regular, smooth, sharp, and bilinear filters; reduced four-sample kernels; horizontal, vertical, and separable two-dimensional convolution; exact AV1 rounding; 8/10/12-bit clipping; padded reference origins; and guarded destination strides. The operator contract and concrete operator files mirror JPEG color conversion, and `FeatureTestRunner` verifies normal, AVX-512-disabled, AVX-disabled, and scalar execution against an independent fixed-point oracle. - - [ ] Decode and connect reference indices, motion vectors, compound prediction, inter-intra prediction, masked blending, warped motion, global motion, and OBMC through reconstructed reference planes. + - [ ] Decode temporal segmentation prediction and consume the retained segmentation map when `segmentation_update_map == 1` and temporal update is enabled. + - [ ] Decode and connect inter-block reference indices, motion vectors, compound prediction, inter-intra prediction, masked blending, warped motion, global motion, and OBMC through reconstructed reference planes. The current tile mode reader accepts intra frames only. - [ ] Verify every connected inter mode and filter with independently encoded dependent-layer AV1 image-item fixtures and exact native-plane comparisons. - [ ] Lossless and high-bit-depth reconstruction with correct clipping and intermediate precision. - [x] Route lossless 4x4 blocks through allocation-free reversible inverse Walsh-Hadamard reconstruction for 8/10/12-bit samples, including the DC-only specialization, `Vector128` production traversal, scalar fallback, exact clipping, and `FeatureTestRunner` parity. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs index e8cc52496..5f1424c33 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamReader.cs @@ -74,7 +74,7 @@ internal ref struct Av1BitStreamReader /// Reads the next encoded bit as a Boolean value. /// /// for one; otherwise, . - internal bool ReadBoolean() => this.ReadLiteral(1) > 0; + public bool ReadBoolean() => this.ReadLiteral(1) > 0; /// /// Reads an AV1 little-endian base-128 value from a byte-aligned position. @@ -159,6 +159,21 @@ internal ref struct Av1BitStreamReader return (v << 1) - m + this.ReadLiteral(1); } + /// + /// Reads a finite subexponential value recentered around a signed reference value. + /// + /// One greater than the maximum absolute value in the signed domain. + /// The bit width of the first subexponential group. + /// The signed reference value around which smaller codewords are concentrated. + /// A decoded value in the inclusive range from minus plus one through + /// minus one. + public int ReadSignedReferenceSubexponential(int valueMagnitude, int groupBitCount, int reference) + { + int shiftedReference = reference + valueMagnitude - 1; + int scaledValueCount = (valueMagnitude << 1) - 1; + return this.ReadReferenceSubexponential(scaledValueCount, groupBitCount, shiftedReference) - valueMagnitude + 1; + } + /// /// Reads a fixed-width two's-complement signed integer. /// @@ -226,4 +241,74 @@ internal ref struct Av1BitStreamReader this.Skip(byteCount << 3); return payload; } + + /// + /// Reads a finite subexponential value and inverse-recenters it around an unsigned reference value. + /// + /// The number of values in the finite domain. + /// The bit width of the first subexponential group. + /// The reference value within the finite domain. + /// The decoded value in the range zero through minus one. + private int ReadReferenceSubexponential(int valueCount, int groupBitCount, int reference) + { + int value = this.ReadSubexponential(valueCount, groupBitCount); + + // Recentering enumerates values by increasing distance from the reference. References in the upper half use + // the mirrored domain so the shorter side of the finite range always participates in the alternating mapping. + if ((reference << 1) <= valueCount) + { + return InverseRecenter(reference, value); + } + + return valueCount - 1 - InverseRecenter(valueCount - 1 - reference, value); + } + + /// + /// Reads one value from a finite subexponential code. + /// + /// The number of values in the finite domain. + /// The bit width of the first subexponential group. + /// The decoded zero-based value. + private int ReadSubexponential(int valueCount, int groupBitCount) + { + int groupIndex = 0; + int groupStart = 0; + while (true) + { + // AV1 keeps the first two groups at width k and then doubles each following group. Once fewer than three + // groups remain, the non-symmetric code consumes the exact finite tail without introducing unused values. + int bitCount = groupIndex == 0 ? groupBitCount : groupBitCount + groupIndex - 1; + int groupSize = 1 << bitCount; + if (valueCount <= groupStart + (3 * groupSize)) + { + return (int)this.ReadNonSymmetric((uint)(valueCount - groupStart)) + groupStart; + } + + if (!this.ReadBoolean()) + { + return (int)this.ReadLiteral(bitCount) + groupStart; + } + + groupIndex++; + groupStart += groupSize; + } + } + + /// + /// Maps a nonnegative code value around a nonnegative reference value. + /// + /// The recentering reference. + /// The coded nonnegative value. + /// The inverse-recentered value. + private static int InverseRecenter(int reference, int value) + { + // Codes within twice the reference alternate above and below it: even values select the upper side and odd + // values select the lower side. Larger codes lie beyond the lower-side range and map directly to the tail. + if (value > (reference << 1)) + { + return value; + } + + return (value & 1) == 0 ? (value >> 1) + reference : reference - ((value + 1) >> 1); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index b30b5de9c..89379f245 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -2,8 +2,11 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1.Color; +using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain; +using SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Metadata; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; @@ -13,7 +16,7 @@ using SixLabors.ImageSharp.PixelFormats.Utils; namespace SixLabors.ImageSharp.Formats.Heif.Av1; /// -/// Decodes one AV1 still-image elementary stream into an ImageSharp image. +/// Decodes one bounded AV1 image payload into an ImageSharp image. /// internal sealed class Av1Decoder : IAv1TileReader, IDisposable { @@ -27,6 +30,37 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable /// private readonly Configuration configuration; + /// + /// The reconstructed references and selected presentation output owned by the current bounded decode session. + /// + private readonly Av1ReferenceFrameStore referenceFrames = new(); + + /// + /// The frame-base, tile-working, and published entropy contexts created for the first coded frame and then reused + /// for this bounded decoder session. + /// + private Av1FrameEntropyContexts? entropyContexts; + + /// + /// The coded sequence governing the active reference map and reusable entropy session. + /// + private ObuSequenceHeader? entropySequenceHeader; + + /// + /// The item codec configuration validated before reconstructing a completed frame. + /// + private Av1CodecConfiguration? codecConfiguration; + + /// + /// The container color description applied before reconstructing a completed frame. + /// + private CicpProfile? containerColorProfile; + + /// + /// The sequence header already validated for the current bounded payload. + /// + private ObuSequenceHeader? validatedSequenceHeader; + /// /// The tile parser shared by all tile groups in the current frame. /// @@ -49,26 +83,26 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable public Av1Decoder(Configuration configuration, byte operatingPointIndex) { this.configuration = configuration; - this.obuReader = new(operatingPointIndex); + this.obuReader = new(operatingPointIndex, this.referenceFrames); } /// - /// Gets the decoded frame header, or before the stream provides one. + /// Gets the final retained shown-frame header, or before a shown frame completes. /// public ObuFrameHeader? FrameHeader { get; private set; } /// - /// Gets the decoded sequence header, or before the stream provides one. + /// Gets the sequence header governing the final retained shown frame, or before one completes. /// public ObuSequenceHeader? SequenceHeader { get; private set; } /// - /// Gets the tile and superblock state for the decoded frame, or before tile parsing completes. + /// Gets the tile and superblock state for the final retained shown frame, or before one completes. /// public Av1FrameInfo? FrameInfo { get; private set; } /// - /// Decodes an AV1 still-image elementary stream. + /// Decodes a bounded AV1 image payload and presents its final shown frame. /// /// The destination pixel type. /// The complete AV1 elementary-stream payload. @@ -196,7 +230,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable } /// - /// Parses and reconstructs one AV1 frame while retaining its native component planes for the caller. + /// Parses every coded frame in an AV1 payload and returns the final shown frame's native component planes. /// /// The complete AV1 elementary-stream payload. /// @@ -211,69 +245,21 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable Av1CodecConfiguration? codecConfiguration, out CicpProfile effectiveColorProfile) { + this.codecConfiguration = codecConfiguration; + this.containerColorProfile = containerColorProfile; + this.validatedSequenceHeader = null; + this.SequenceHeader = null; + this.FrameHeader = null; + this.FrameInfo = null; Av1BitStreamReader reader = new(buffer); - this.obuReader.ReadAll(ref reader, buffer.Length, () => this, false); - Guard.NotNull(this.tileReader, nameof(this.tileReader)); - Guard.NotNull(this.SequenceHeader, nameof(this.SequenceHeader)); - Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader)); - codecConfiguration?.Validate(this.SequenceHeader); - - if (containerColorProfile is not null) - { - ObuColorConfig colorConfig = this.SequenceHeader.ColorConfig; - ObuColorPrimaries containerColorPrimaries = (ObuColorPrimaries)containerColorProfile.ColorPrimaries; - ObuTransferCharacteristics containerTransferCharacteristics = - (ObuTransferCharacteristics)containerColorProfile.TransferCharacteristics; - - ObuMatrixCoefficients containerMatrixCoefficients = - (ObuMatrixCoefficients)containerColorProfile.MatrixCoefficients; - - // AV1-ISOBMFF permits nclx to supply only bitstream fields explicitly coded as unspecified. A - // different specified value is a conformance error rather than a container-level color override. - if (colorConfig.ColorPrimaries == ObuColorPrimaries.Unspecified) - { - colorConfig.ColorPrimaries = containerColorPrimaries; - } - else if (colorConfig.ColorPrimaries != containerColorPrimaries) - { - throw new InvalidImageContentException("The HEIF CICP color primaries do not match the AV1 sequence header."); - } - - if (colorConfig.TransferCharacteristics == ObuTransferCharacteristics.Unspecified) - { - colorConfig.TransferCharacteristics = containerTransferCharacteristics; - } - else if (colorConfig.TransferCharacteristics != containerTransferCharacteristics) - { - throw new InvalidImageContentException("The HEIF CICP transfer characteristics do not match the AV1 sequence header."); - } - - if (colorConfig.MatrixCoefficients == ObuMatrixCoefficients.Unspecified) - { - colorConfig.MatrixCoefficients = containerMatrixCoefficients; - } - else if (colorConfig.MatrixCoefficients != containerMatrixCoefficients) - { - throw new InvalidImageContentException("The HEIF CICP matrix coefficients do not match the AV1 sequence header."); - } - - if (colorConfig.ColorRange != containerColorProfile.FullRange) - { - throw new InvalidImageContentException("The HEIF CICP color range does not match the AV1 sequence header."); - } - } - - this.FrameInfo = this.tileReader.FrameInfo; - Av1FrameBuffer frameBuffer = new( - this.configuration, - this.SequenceHeader, - this.SequenceHeader.ColorConfig.GetColorFormat(), - false); try { - using Av1FrameDecoder frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, frameBuffer); - frameDecoder.DecodeFrame(); + this.obuReader.ReadAll(ref reader, buffer.Length, () => this, false); + + Guard.NotNull(this.referenceFrames.OutputFrame, nameof(this.referenceFrames.OutputFrame)); + Guard.NotNull(this.SequenceHeader, nameof(this.SequenceHeader)); + Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader)); // Preserve the effective CICP description used for conversion, including container values that legally // supplied unspecified bitstream fields. This also exposes bitstream-only color metadata to callers. @@ -284,13 +270,31 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable (byte)effectiveColorConfig.MatrixCoefficients, effectiveColorConfig.ColorRange); - return frameBuffer; + using Av1ReferenceFrame outputFrame = this.referenceFrames.TakeOutput(); + return outputFrame.TakeFrameBuffer(); } catch { - frameBuffer.Dispose(); + // A failed frame may own pooled neighbor contexts while earlier layers own reconstructed references and + // published CDF snapshots. None can be reused after a non-transactional frame transition has failed. + this.tileReader?.Dispose(); + this.tileReader = null; + this.obuReader.Reset(); + this.entropyContexts?.Reset(); + this.entropySequenceHeader = null; + this.SequenceHeader = null; + this.FrameHeader = null; + this.FrameInfo = null; throw; } + finally + { + // Validation inputs belong to this bounded decode call. Completed native buffers retain no references to + // either description, so releasing them here prevents a reused decoder from observing stale item state. + this.codecConfiguration = null; + this.containerColorProfile = null; + this.validatedSequenceHeader = null; + } } /// @@ -302,24 +306,214 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable { if (this.tileReader is null) { - this.SequenceHeader = this.obuReader.SequenceHeader; - this.FrameHeader = this.obuReader.FrameHeader; - Guard.NotNull(this.SequenceHeader, nameof(this.SequenceHeader)); - Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader)); + ObuSequenceHeader? sequenceHeader = this.obuReader.SequenceHeader; + ObuFrameHeader? frameHeader = this.obuReader.FrameHeader; + Guard.NotNull(sequenceHeader, nameof(sequenceHeader)); + Guard.NotNull(frameHeader, nameof(frameHeader)); + + if (!ReferenceEquals(this.entropySequenceHeader, sequenceHeader)) + { + if (this.entropySequenceHeader is not null) + { + // A coded-sequence boundary invalidates both sample references and their retained CDF snapshots. + // Returned snapshot graphs stay decoder-local and can be overwritten for the new sequence. + this.referenceFrames.Reset(); + this.entropyContexts?.Reset(); + } + + this.entropySequenceHeader = sequenceHeader; + } + + Av1FrameEntropyContext? primaryReferenceContext = null; + byte? primaryReferenceSlot = frameHeader.PrimaryReferenceSlot; + if (primaryReferenceSlot is not null) + { + // The uncompressed-header parser validates slot occupancy. Entropy ownership is checked here because + // only the reconstructed frame owner knows whether that slot retained a completed CDF snapshot. + Av1ReferenceFrame? primaryReference = this.referenceFrames.Resolve(primaryReferenceSlot.Value); + if (primaryReference is null || primaryReference.EntropyContext is null) + { + throw new InvalidImageContentException("The AV1 primary reference has no retained entropy context."); + } + + primaryReferenceContext = primaryReference.EntropyContext; + } // Every tile group in a frame contributes to the same mode-info and coefficient state. - this.tileReader = new Av1TileReader(this.configuration, this.SequenceHeader, this.FrameHeader); + Av1FrameEntropyContexts entropyContexts = + this.entropyContexts ??= new(frameHeader.QuantizationParameters.BaseQIndex); + + this.tileReader = new Av1TileReader( + this.configuration, + sequenceHeader, + frameHeader, + entropyContexts, + primaryReferenceContext, + this.referenceFrames); } this.tileReader.ReadTile(tileData, tileNum); } /// - /// Releases the tile reader and its frame-scoped parsing storage. + /// Reconstructs a frame after all of its tile payloads have been parsed. + /// + public void CompleteFrame() + { + Av1TileReader tileReader = this.tileReader!; + ObuSequenceHeader sequenceHeader = this.obuReader.SequenceHeader!; + ObuFrameHeader frameHeader = this.obuReader.FrameHeader!; + Av1FrameBuffer? frameBuffer = null; + Av1FrameBuffer? presentationBuffer = null; + + try + { + if (!ReferenceEquals(this.validatedSequenceHeader, sequenceHeader)) + { + this.codecConfiguration?.Validate(sequenceHeader); + CicpProfile? colorProfile = this.containerColorProfile; + + if (colorProfile is not null) + { + ObuColorConfig colorConfig = sequenceHeader.ColorConfig; + ObuColorPrimaries containerColorPrimaries = (ObuColorPrimaries)colorProfile.ColorPrimaries; + ObuTransferCharacteristics containerTransferCharacteristics = + (ObuTransferCharacteristics)colorProfile.TransferCharacteristics; + + ObuMatrixCoefficients containerMatrixCoefficients = + (ObuMatrixCoefficients)colorProfile.MatrixCoefficients; + + // AV1-ISOBMFF permits nclx to supply only bitstream fields explicitly coded as unspecified. A + // different specified value is a conformance error rather than a container-level color override. + if (colorConfig.ColorPrimaries == ObuColorPrimaries.Unspecified) + { + colorConfig.ColorPrimaries = containerColorPrimaries; + } + else if (colorConfig.ColorPrimaries != containerColorPrimaries) + { + throw new InvalidImageContentException("The HEIF CICP color primaries do not match the AV1 sequence header."); + } + + if (colorConfig.TransferCharacteristics == ObuTransferCharacteristics.Unspecified) + { + colorConfig.TransferCharacteristics = containerTransferCharacteristics; + } + else if (colorConfig.TransferCharacteristics != containerTransferCharacteristics) + { + throw new InvalidImageContentException("The HEIF CICP transfer characteristics do not match the AV1 sequence header."); + } + + if (colorConfig.MatrixCoefficients == ObuMatrixCoefficients.Unspecified) + { + colorConfig.MatrixCoefficients = containerMatrixCoefficients; + } + else if (colorConfig.MatrixCoefficients != containerMatrixCoefficients) + { + throw new InvalidImageContentException("The HEIF CICP matrix coefficients do not match the AV1 sequence header."); + } + + if (colorConfig.ColorRange != colorProfile.FullRange) + { + throw new InvalidImageContentException("The HEIF CICP color range does not match the AV1 sequence header."); + } + } + + // The same sequence header governs subsequent layered frames until another header OBU replaces it. + // Retaining the validated reference prevents repeated item/color checks in the common layered case. + this.validatedSequenceHeader = sequenceHeader; + } + + Av1FrameInfo frameInfo = tileReader.FrameInfo; + frameBuffer = new Av1FrameBuffer( + this.configuration, + sequenceHeader, + sequenceHeader.ColorConfig.GetColorFormat(), + false); + + using Av1FrameDecoder frameDecoder = new(sequenceHeader, frameHeader, frameInfo, frameBuffer); + frameDecoder.DecodeFrame(); + + bool retainsReference = (frameHeader.RefreshFrameFlags & byte.MaxValue) != 0; + if (retainsReference) + { + // Motion compensation may address any clamped position inside the decoder border. Extending once after + // all in-loop filters lets every later block use the full padded span without per-prediction edge copies. + Av1ReferenceFrameBorder.Extend(frameBuffer); + } + + bool needsSeparatePresentation = frameHeader.ShowFrame && frameHeader.FilmGrainParameters.ApplyGrain && retainsReference; + if (needsSeparatePresentation) + { + presentationBuffer = new Av1FrameBuffer( + this.configuration, + sequenceHeader, + sequenceHeader.ColorConfig.GetColorFormat(), + false); + + // Film grain must never contaminate a decoded reference. A shown frame that is also refreshed therefore + // receives one allocator-owned presentation copy; frames with no reference role are grained in place. + frameBuffer.CopyTo(presentationBuffer); + } + + Av1FrameBuffer grainTarget = presentationBuffer ?? frameBuffer; + if (frameHeader.ShowFrame && frameHeader.FilmGrainParameters.ApplyGrain) + { + Av1FilmGrainDecoder filmGrainDecoder = new(sequenceHeader, frameHeader, grainTarget); + filmGrainDecoder.DecodeFrame(); + } + + Av1ReferenceFrame referenceFrame; + if (retainsReference) + { + Av1FrameEntropyContexts entropyContexts = this.entropyContexts!; + Av1FrameEntropyContext entropySnapshot = entropyContexts.RentPublishedSnapshot(); + referenceFrame = new(frameBuffer, frameHeader, frameInfo, entropySnapshot, entropyContexts); + } + else + { + // Presentation-only frames can never become primary references, so they own no unused CDF graph. + referenceFrame = new(frameBuffer, frameHeader, frameInfo); + } + + frameBuffer = null; + + if (!this.referenceFrames.Commit(frameHeader.RefreshFrameFlags, referenceFrame, frameHeader.ShowFrame && !needsSeparatePresentation)) + { + referenceFrame.Dispose(); + } + + if (presentationBuffer is not null) + { + Av1ReferenceFrame presentationFrame = new(presentationBuffer, frameHeader, frameInfo); + presentationBuffer = null; + this.referenceFrames.CommitOutput(presentationFrame); + } + + if (frameHeader.ShowFrame) + { + this.SequenceHeader = sequenceHeader; + this.FrameHeader = frameHeader; + this.FrameInfo = frameInfo; + } + } + finally + { + // A non-shown frame or failed reconstruction never escapes this callback. FrameInfo uses managed storage, + // so it remains inspectable for a retained frame after the pooled entropy-neighbor contexts are returned. + presentationBuffer?.Dispose(); + frameBuffer?.Dispose(); + tileReader.Dispose(); + this.tileReader = null; + } + } + + /// + /// Releases the current tile parser, reference map, and retained presentation output. /// public void Dispose() { this.tileReader?.Dispose(); this.tileReader = null; + this.referenceFrames.Dispose(); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs index 3409d2dc0..f3a79566c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1FrameBuffer.cs @@ -108,19 +108,29 @@ internal class Av1FrameBuffer : IDisposable this.BufferY = null; this.BufferCb = null; this.BufferCr = null; - if ((bufferEnableMask & PictureBufferYFlag) != 0) + try { - this.BufferY = configuration.MemoryAllocator.Allocate2D(strideY * this.storageElementsPerSample, heightY); + if ((bufferEnableMask & PictureBufferYFlag) != 0) + { + this.BufferY = configuration.MemoryAllocator.Allocate2D(strideY * this.storageElementsPerSample, heightY); + } + + if ((bufferEnableMask & PictureBufferCbFlag) != 0) + { + this.BufferCb = configuration.MemoryAllocator.Allocate2D(strideChroma * this.storageElementsPerSample, heightChroma); + } + + if ((bufferEnableMask & PictureBufferCrFlag) != 0) + { + this.BufferCr = configuration.MemoryAllocator.Allocate2D(strideChroma * this.storageElementsPerSample, heightChroma); + } } - - if ((bufferEnableMask & PictureBufferCbFlag) != 0) - { - this.BufferCb = configuration.MemoryAllocator.Allocate2D(strideChroma * this.storageElementsPerSample, heightChroma); - } - - if ((bufferEnableMask & PictureBufferCrFlag) != 0) + catch { - this.BufferCr = configuration.MemoryAllocator.Allocate2D(strideChroma * this.storageElementsPerSample, heightChroma); + // Construction publishes the owner only after every required plane has been rented. Release earlier planes + // here because a later allocation failure leaves no constructed frame buffer for the caller to dispose. + this.Dispose(); + throw; } } @@ -204,6 +214,34 @@ internal class Av1FrameBuffer : IDisposable /// public MemoryAllocator MemoryAllocator { get; } + /// + /// Copies the complete padded sample planes and active picture geometry to another compatible frame buffer. + /// + /// The frame buffer receiving the copied reconstruction. + public void CopyTo(Av1FrameBuffer destination) + { + // Copy each contiguous allocation so the runtime can use its optimized bulk-memory path. Film-grain + // presentation consumes right and bottom padding for odd dimensions, so copying only visible rows would leave + // part of the independently owned presentation surface undefined. + this.BufferY!.DangerousGetSingleSpan().CopyTo(destination.BufferY!.DangerousGetSingleSpan()); + Buffer2D? chromaBlue = this.BufferCb; + if (chromaBlue is not null) + { + chromaBlue.DangerousGetSingleSpan().CopyTo(destination.BufferCb!.DangerousGetSingleSpan()); + this.BufferCr!.DangerousGetSingleSpan().CopyTo(destination.BufferCr!.DangerousGetSingleSpan()); + } + + destination.StartPosition = this.StartPosition; + destination.OriginX = this.OriginX; + destination.OriginY = this.OriginY; + destination.Width = this.Width; + destination.Height = this.Height; + destination.MaxWidth = this.MaxWidth; + destination.MaxHeight = this.MaxHeight; + destination.BitDepth = this.BitDepth; + destination.ColorFormat = this.ColorFormat; + } + /// /// Releases the owned luma and chroma plane allocations. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DefaultDistributions.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DefaultDistributions.cs index 800bda9e0..f40d7de25 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DefaultDistributions.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DefaultDistributions.cs @@ -326,9 +326,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 16 coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 16 coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti16 => + private static Av1Distribution[][][] EndOfBlockFlagMulti16 { get; } = [ [ [new(840, 1039, 1980, 4895), new(370, 671, 1883, 4471)], @@ -349,9 +350,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 32 coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 32 coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti32 => + private static Av1Distribution[][][] EndOfBlockFlagMulti32 { get; } = [ [ [new(400, 520, 977, 2102, 6542), new(210, 405, 1315, 3326, 7537)], @@ -372,9 +374,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 64 coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 64 coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti64 => + private static Av1Distribution[][][] EndOfBlockFlagMulti64 { get; } = [ [ [new(329, 498, 1101, 1784, 3265, 7758), new(335, 730, 1459, 5494, 8755, 12997)], @@ -395,9 +398,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 128 coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 128 coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti128 => + private static Av1Distribution[][][] EndOfBlockFlagMulti128 { get; } = [ [ [new(219, 482, 1140, 2091, 3680, 6028, 12586), new(371, 699, 1254, 4830, 9479, 12562, 17497)], @@ -418,9 +422,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 256 coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 256 coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti256 => + private static Av1Distribution[][][] EndOfBlockFlagMulti256 { get; } = [ [ [ @@ -465,9 +470,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 512 coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 512 coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti512 => + private static Av1Distribution[][][] EndOfBlockFlagMulti512 { get; } = [ [ [ @@ -512,9 +518,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block token distributions for transforms containing at most 1024 coded coefficients, indexed by quantizer, plane, and transform-class contexts. + /// Gets the end-of-block token distributions for transforms containing at most 1024 coded coefficients, indexed by + /// quantizer, plane, and transform-class contexts. /// - private static Av1Distribution[][][] EndOfBlockFlagMulti1024 => + private static Av1Distribution[][][] EndOfBlockFlagMulti1024 { get; } = [ [ [ @@ -559,9 +566,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the coefficient base-range distributions indexed by quantizer, transform-size, plane, and base-range contexts. + /// Gets the coefficient base-range distributions indexed by quantizer, transform-size, plane, and base-range + /// contexts. /// - private static Av1Distribution[][][][] CoefficientsBaseRange => + private static Av1Distribution[][][][] CoefficientsBaseRange { get; } = [ [ [ @@ -974,9 +982,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the coefficient base-level distributions indexed by quantizer, transform-size, plane, and nonzero-map contexts. + /// Gets the coefficient base-level distributions indexed by quantizer, transform-size, plane, and nonzero-map + /// contexts. /// - private static Av1Distribution[][][][] CoefficientsBase => + private static Av1Distribution[][][][] CoefficientsBase { get; } = [ [ [ @@ -1669,9 +1678,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the final-nonzero coefficient base-level distributions indexed by quantizer, transform-size, plane, and end-of-block contexts. + /// Gets the final-nonzero coefficient base-level distributions indexed by quantizer, transform-size, plane, and + /// end-of-block contexts. /// - private static Av1Distribution[][][][] BaseEndOfBlock => + private static Av1Distribution[][][][] BaseEndOfBlock { get; } = [ [ [ @@ -1766,7 +1776,7 @@ internal static class Av1DefaultDistributions /// /// Gets the DC coefficient-sign distributions indexed by quantizer, plane, and neighboring sign contexts. /// - private static Av1Distribution[][][] DcSign => + private static Av1Distribution[][][] DcSign { get; } = [ [ [new(128 * 125), new(128 * 102), new(128 * 147)], @@ -1789,7 +1799,7 @@ internal static class Av1DefaultDistributions /// /// Gets the transform-block skip distributions indexed by quantizer, transform-size, and neighboring skip contexts. /// - private static Av1Distribution[][][] TransformBlockSkip => + private static Av1Distribution[][][] TransformBlockSkip { get; } = [ [ [ @@ -1882,9 +1892,10 @@ internal static class Av1DefaultDistributions ]; /// - /// Gets the end-of-block extra-bit distributions indexed by quantizer, transform-size, plane, and padded token contexts. + /// Gets the end-of-block extra-bit distributions indexed by quantizer, transform-size, plane, and padded token + /// contexts. /// - private static Av1Distribution[][][][] EndOfBlockExtra => + private static Av1Distribution[][][][] EndOfBlockExtra { get; } = [ [ [ @@ -2419,13 +2430,13 @@ internal static class Av1DefaultDistributions int qContext = GetQContext(baseQIndex); return [ - EndOfBlockFlagMulti16[qContext], - EndOfBlockFlagMulti32[qContext], - EndOfBlockFlagMulti64[qContext], - EndOfBlockFlagMulti128[qContext], - EndOfBlockFlagMulti256[qContext], - EndOfBlockFlagMulti512[qContext], - EndOfBlockFlagMulti1024[qContext], + Av1Distribution.CreateCopy(EndOfBlockFlagMulti16[qContext]), + Av1Distribution.CreateCopy(EndOfBlockFlagMulti32[qContext]), + Av1Distribution.CreateCopy(EndOfBlockFlagMulti64[qContext]), + Av1Distribution.CreateCopy(EndOfBlockFlagMulti128[qContext]), + Av1Distribution.CreateCopy(EndOfBlockFlagMulti256[qContext]), + Av1Distribution.CreateCopy(EndOfBlockFlagMulti512[qContext]), + Av1Distribution.CreateCopy(EndOfBlockFlagMulti1024[qContext]), ]; } @@ -2435,7 +2446,7 @@ internal static class Av1DefaultDistributions /// The frame base quantizer index. /// The distributions indexed by transform-size, plane, and base-range contexts. public static Av1Distribution[][][] GetCoefficientsBaseRange(int baseQIndex) - => CoefficientsBaseRange[GetQContext(baseQIndex)]; + => Av1Distribution.CreateCopy(CoefficientsBaseRange[GetQContext(baseQIndex)]); /// /// Gets the coefficient base-level distributions for a base quantizer. @@ -2443,7 +2454,7 @@ internal static class Av1DefaultDistributions /// The frame base quantizer index. /// The distributions indexed by transform-size, plane, and nonzero-map contexts. public static Av1Distribution[][][] GetCoefficientsBase(int baseQIndex) - => CoefficientsBase[GetQContext(baseQIndex)]; + => Av1Distribution.CreateCopy(CoefficientsBase[GetQContext(baseQIndex)]); /// /// Gets the final-nonzero coefficient base-level distributions for a base quantizer. @@ -2451,7 +2462,7 @@ internal static class Av1DefaultDistributions /// The frame base quantizer index. /// The distributions indexed by transform-size, plane, and end-of-block contexts. public static Av1Distribution[][][] GetBaseEndOfBlock(int baseQIndex) - => BaseEndOfBlock[GetQContext(baseQIndex)]; + => Av1Distribution.CreateCopy(BaseEndOfBlock[GetQContext(baseQIndex)]); /// /// Gets the DC sign distributions for a base quantizer. @@ -2459,7 +2470,7 @@ internal static class Av1DefaultDistributions /// The frame base quantizer index. /// The distributions indexed by plane and neighboring sign context. public static Av1Distribution[][] GetDcSign(int baseQIndex) - => DcSign[GetQContext(baseQIndex)]; + => Av1Distribution.CreateCopy(DcSign[GetQContext(baseQIndex)]); /// /// Gets the transform-block skip distributions for a base quantizer. @@ -2467,7 +2478,7 @@ internal static class Av1DefaultDistributions /// The frame base quantizer index. /// The distributions indexed by transform-size and neighboring skip context. public static Av1Distribution[][] GetTransformBlockSkip(int baseQIndex) - => TransformBlockSkip[GetQContext(baseQIndex)]; + => Av1Distribution.CreateCopy(TransformBlockSkip[GetQContext(baseQIndex)]); /// /// Gets the end-of-block extra-bit distributions for a base quantizer. @@ -2475,7 +2486,7 @@ internal static class Av1DefaultDistributions /// The frame base quantizer index. /// The distributions indexed by transform-size, plane, and end-of-block token context. public static Av1Distribution[][][] GetEndOfBlockExtra(int baseQIndex) - => EndOfBlockExtra[GetQContext(baseQIndex)]; + => Av1Distribution.CreateCopy(EndOfBlockExtra[GetQContext(baseQIndex)]); /// /// Maps a base quantizer index to one of the four AV1 coefficient-probability initialization bands. diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DisplacementVectorContext.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DisplacementVectorContext.cs index 6fbc64857..23483c4b5 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DisplacementVectorContext.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1DisplacementVectorContext.cs @@ -35,6 +35,27 @@ internal sealed class Av1DisplacementVectorContext /// private readonly Component horizontal = new(); + /// + /// Replaces every displacement-vector distribution with state copied from another context. + /// + /// The displacement-vector context state to copy. + public void CopyFrom(Av1DisplacementVectorContext source) + { + this.joint.CopyFrom(source.joint); + this.vertical.CopyFrom(source.vertical); + this.horizontal.CopyFrom(source.horizontal); + } + + /// + /// Resets every observation count used to adapt displacement-vector distributions. + /// + public void ResetUpdateCounts() + { + this.joint.ResetUpdateCount(); + this.vertical.ResetUpdateCounts(); + this.horizontal.ResetUpdateCounts(); + } + /// /// Reads an integer displacement vector relative to a spatially derived reference. /// @@ -105,6 +126,37 @@ internal sealed class Av1DisplacementVectorContext new(24576), new(28672), new(29952), new(29952), new(30720) ]; + /// + /// Replaces every component distribution with state copied from another component. + /// + /// The component state to copy. + public void CopyFrom(Component source) + { + this.magnitudeClass.CopyFrom(source.magnitudeClass); + this.sign.CopyFrom(source.sign); + this.classZero.CopyFrom(source.classZero); + + for (int bit = 0; bit < this.offsetBits.Length; bit++) + { + this.offsetBits[bit].CopyFrom(source.offsetBits[bit]); + } + } + + /// + /// Resets every observation count used to adapt one component's distributions. + /// + public void ResetUpdateCounts() + { + this.magnitudeClass.ResetUpdateCount(); + this.sign.ResetUpdateCount(); + this.classZero.ResetUpdateCount(); + + for (int bit = 0; bit < this.offsetBits.Length; bit++) + { + this.offsetBits[bit].ResetUpdateCount(); + } + } + /// /// Reads one signed integer-precision component. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1Distribution.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1Distribution.cs index a2feca792..f92a5bde1 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1Distribution.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1Distribution.cs @@ -11,22 +11,22 @@ internal class Av1Distribution /// /// The exclusive upper bound of the Q15 probability domain. /// - internal const int ProbabilityTop = 1 << ProbabilityBitCount; + public const int ProbabilityTop = 1 << ProbabilityBitCount; /// /// The minimum sub-range reserved for each symbol during range coding. /// - internal const int ProbabilityMinimum = 4; + public const int ProbabilityMinimum = 4; /// /// The shift that converts stored Q15 cumulative values to the range-coder precision. /// - internal const int CdfShift = 15 - ProbabilityBitCount; + public const int CdfShift = 15 - ProbabilityBitCount; /// /// The precision reduction applied before multiplying a cumulative value by the coding range. /// - internal const int ProbabilityShift = 6; + public const int ProbabilityShift = 6; /// /// The number of fractional bits in a stored cumulative probability. @@ -320,6 +320,23 @@ internal class Av1Distribution /// A distribution initialized with the same probabilities and update count. public Av1Distribution CreateCopy() => new(this); + /// + /// Replaces the probability and adaptation state with the state of another distribution having the same alphabet. + /// + /// The distribution state to copy. + public void CopyFrom(Av1Distribution source) + { + // Entropy contexts are created from the same fixed default table shape. Copy only mutable state so resetting a + // working tile never allocates or replaces the distribution objects referenced by the symbol decoder. + source.probabilities.AsSpan().CopyTo(this.probabilities); + this.updateCount = source.updateCount; + } + + /// + /// Resets the observation count that controls the adaptive update rate without changing probability thresholds. + /// + public void ResetUpdateCount() => this.updateCount = 0; + /// /// Creates independently adaptable copies of a distribution array. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs new file mode 100644 index 000000000..f0c1a3f6b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs @@ -0,0 +1,519 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; + +/// +/// Owns the adaptive AV1 distributions currently implemented by the frame and tile syntax decoders. +/// +/// +/// One frame context supplies the initial state copied into every tile context. Each tile adapts an independent working +/// copy, and only the tile selected by context_update_tile_id supplies the completed frame snapshot. +/// +internal sealed class Av1FrameEntropyContext +{ + /// + /// The inclusive upper bound of the first AV1 coefficient-probability quantizer band. + /// + private const int FirstQuantizerBandMaximum = 20; + + /// + /// The inclusive upper bound of the second AV1 coefficient-probability quantizer band. + /// + private const int SecondQuantizerBandMaximum = 60; + + /// + /// The inclusive upper bound of the third AV1 coefficient-probability quantizer band. + /// + private const int ThirdQuantizerBandMaximum = 120; + + /// + /// The immutable normative contexts used to restore reusable frame state without rebuilding distribution graphs. + /// + private static readonly Av1FrameEntropyContext[] DefaultPrototypes = + [ + new((byte)0), + new((byte)1), + new((byte)2), + new((byte)3) + ]; + + /// + /// Initializes a new instance of the class from the normative default + /// distributions selected by a frame quantizer index. + /// + /// The frame base quantizer index selecting coefficient distribution defaults. + public Av1FrameEntropyContext(int qIndex) + : this(DefaultPrototypes[GetQContext(qIndex)]) + { + } + + /// + /// Initializes a new instance of the class as an immutable normative prototype. + /// + /// The zero-based coefficient-probability quantizer band. + private Av1FrameEntropyContext(byte qContext) + { + int qIndex = qContext switch + { + 0 => 0, + 1 => FirstQuantizerBandMaximum + 1, + 2 => SecondQuantizerBandMaximum + 1, + _ => ThirdQuantizerBandMaximum + 1 + }; + + // Every default-distribution accessor constructs independently mutable state. Retaining those returned + // graphs directly confines generated-table construction to the four process-wide quantizer-band prototypes. + this.IntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy; + this.DisplacementVector = new(); + this.SwitchableRestoration = Av1DefaultDistributions.SwitchableRestoration; + this.WienerRestoration = Av1DefaultDistributions.WienerRestoration; + this.SgrProjectionRestoration = Av1DefaultDistributions.SgrProjectionRestoration; + this.PaletteYMode = Av1DefaultDistributions.PaletteYMode; + this.PaletteUvMode = Av1DefaultDistributions.PaletteUvMode; + this.PaletteYSize = Av1DefaultDistributions.PaletteYSize; + this.PaletteUvSize = Av1DefaultDistributions.PaletteUvSize; + this.PaletteYColorIndex = Av1DefaultDistributions.PaletteYColorIndex; + this.PaletteUvColorIndex = Av1DefaultDistributions.PaletteUvColorIndex; + this.PartitionTypes = Av1DefaultDistributions.PartitionTypes; + this.KeyFrameYMode = Av1DefaultDistributions.KeyFrameYMode; + this.UvMode = Av1DefaultDistributions.UvMode; + this.Skip = Av1DefaultDistributions.Skip; + this.SkipMode = Av1DefaultDistributions.SkipMode; + this.DeltaLoopFilterAbsolute = Av1DefaultDistributions.DeltaLoopFilterAbsolute; + this.DeltaQuantizerAbsolute = Av1DefaultDistributions.DeltaQuantizerAbsolute; + this.SegmentId = Av1DefaultDistributions.SegmentId; + this.AngleDelta = Av1DefaultDistributions.AngleDelta; + this.FilterIntraMode = Av1DefaultDistributions.FilterIntraMode; + this.FilterIntra = Av1DefaultDistributions.FilterIntra; + this.TransformSize = Av1DefaultDistributions.TransformSize; + this.ChromaFromLumaSign = Av1DefaultDistributions.ChromaFromLumaSign; + this.ChromaFromLumaAlpha = Av1DefaultDistributions.ChromaFromLumaAlpha; + this.IntraExtendedTransform = Av1DefaultDistributions.IntraExtendedTransform; + this.InterExtendedTransform = Av1DefaultDistributions.InterExtendedTransform; + + // Coefficient defaults use one of four quantizer bands. Their array shapes remain fixed, so later tile resets + // copy only thresholds and update counts into this context's already allocated distribution graph. + this.EndOfBlockFlag = Av1DefaultDistributions.GetEndOfBlockFlag(qIndex); + this.CoefficientsBase = Av1DefaultDistributions.GetCoefficientsBase(qIndex); + this.BaseEndOfBlock = Av1DefaultDistributions.GetBaseEndOfBlock(qIndex); + this.DcSign = Av1DefaultDistributions.GetDcSign(qIndex); + this.CoefficientsBaseRange = Av1DefaultDistributions.GetCoefficientsBaseRange(qIndex); + this.TransformBlockSkip = Av1DefaultDistributions.GetTransformBlockSkip(qIndex); + this.EndOfBlockExtra = Av1DefaultDistributions.GetEndOfBlockExtra(qIndex); + } + + /// + /// Initializes a new instance of the class with an independently adaptable copy of a prototype. + /// + /// The prototype or retained context whose state is copied. + private Av1FrameEntropyContext(Av1FrameEntropyContext source) + { + // Session and retained-frame contexts need one mutable graph, not four generated quantizer-band graphs whose + // unused bands are immediately discarded. Deep-copy the already selected prototype shape exactly once. + this.IntraBlockCopy = source.IntraBlockCopy.CreateCopy(); + this.DisplacementVector = new(); + this.DisplacementVector.CopyFrom(source.DisplacementVector); + this.SwitchableRestoration = source.SwitchableRestoration.CreateCopy(); + this.WienerRestoration = source.WienerRestoration.CreateCopy(); + this.SgrProjectionRestoration = source.SgrProjectionRestoration.CreateCopy(); + this.PaletteYMode = Av1Distribution.CreateCopy(source.PaletteYMode); + this.PaletteUvMode = Av1Distribution.CreateCopy(source.PaletteUvMode); + this.PaletteYSize = Av1Distribution.CreateCopy(source.PaletteYSize); + this.PaletteUvSize = Av1Distribution.CreateCopy(source.PaletteUvSize); + this.PaletteYColorIndex = Av1Distribution.CreateCopy(source.PaletteYColorIndex); + this.PaletteUvColorIndex = Av1Distribution.CreateCopy(source.PaletteUvColorIndex); + this.PartitionTypes = Av1Distribution.CreateCopy(source.PartitionTypes); + this.KeyFrameYMode = Av1Distribution.CreateCopy(source.KeyFrameYMode); + this.UvMode = Av1Distribution.CreateCopy(source.UvMode); + this.Skip = Av1Distribution.CreateCopy(source.Skip); + this.SkipMode = Av1Distribution.CreateCopy(source.SkipMode); + this.DeltaLoopFilterAbsolute = source.DeltaLoopFilterAbsolute.CreateCopy(); + this.DeltaQuantizerAbsolute = source.DeltaQuantizerAbsolute.CreateCopy(); + this.SegmentId = Av1Distribution.CreateCopy(source.SegmentId); + this.AngleDelta = Av1Distribution.CreateCopy(source.AngleDelta); + this.FilterIntraMode = source.FilterIntraMode.CreateCopy(); + this.FilterIntra = Av1Distribution.CreateCopy(source.FilterIntra); + this.TransformSize = Av1Distribution.CreateCopy(source.TransformSize); + this.EndOfBlockFlag = Av1Distribution.CreateCopy(source.EndOfBlockFlag); + this.CoefficientsBase = Av1Distribution.CreateCopy(source.CoefficientsBase); + this.BaseEndOfBlock = Av1Distribution.CreateCopy(source.BaseEndOfBlock); + this.DcSign = Av1Distribution.CreateCopy(source.DcSign); + this.CoefficientsBaseRange = Av1Distribution.CreateCopy(source.CoefficientsBaseRange); + this.TransformBlockSkip = Av1Distribution.CreateCopy(source.TransformBlockSkip); + this.EndOfBlockExtra = Av1Distribution.CreateCopy(source.EndOfBlockExtra); + this.ChromaFromLumaSign = source.ChromaFromLumaSign.CreateCopy(); + this.ChromaFromLumaAlpha = Av1Distribution.CreateCopy(source.ChromaFromLumaAlpha); + this.IntraExtendedTransform = Av1Distribution.CreateCopy(source.IntraExtendedTransform); + this.InterExtendedTransform = Av1Distribution.CreateCopy(source.InterExtendedTransform); + } + + /// + /// Gets the intra-block-copy distribution. + /// + public Av1Distribution IntraBlockCopy { get; } + + /// + /// Gets the integer displacement-vector context used by intra-block copy. + /// + public Av1DisplacementVectorContext DisplacementVector { get; } + + /// + /// Gets the switchable loop-restoration distribution. + /// + public Av1Distribution SwitchableRestoration { get; } + + /// + /// Gets the Wiener loop-restoration distribution. + /// + public Av1Distribution WienerRestoration { get; } + + /// + /// Gets the self-guided loop-restoration distribution. + /// + public Av1Distribution SgrProjectionRestoration { get; } + + /// + /// Gets the luma palette-mode distributions. + /// + public Av1Distribution[][] PaletteYMode { get; } + + /// + /// Gets the chroma palette-mode distributions. + /// + public Av1Distribution[] PaletteUvMode { get; } + + /// + /// Gets the luma palette-size distributions. + /// + public Av1Distribution[] PaletteYSize { get; } + + /// + /// Gets the chroma palette-size distributions. + /// + public Av1Distribution[] PaletteUvSize { get; } + + /// + /// Gets the luma palette color-index distributions. + /// + public Av1Distribution[][] PaletteYColorIndex { get; } + + /// + /// Gets the chroma palette color-index distributions. + /// + public Av1Distribution[][] PaletteUvColorIndex { get; } + + /// + /// Gets the partition-type distributions. + /// + public Av1Distribution[] PartitionTypes { get; } + + /// + /// Gets the key-frame luma-mode distributions. + /// + public Av1Distribution[][] KeyFrameYMode { get; } + + /// + /// Gets the chroma intra-mode distributions. + /// + public Av1Distribution[][] UvMode { get; } + + /// + /// Gets the transform-skip distributions. + /// + public Av1Distribution[] Skip { get; } + + /// + /// Gets the skip-mode distributions. + /// + public Av1Distribution[] SkipMode { get; } + + /// + /// Gets the absolute loop-filter delta distribution. + /// + public Av1Distribution DeltaLoopFilterAbsolute { get; } + + /// + /// Gets the absolute quantizer delta distribution. + /// + public Av1Distribution DeltaQuantizerAbsolute { get; } + + /// + /// Gets the spatial segment-identifier distributions. + /// + public Av1Distribution[] SegmentId { get; } + + /// + /// Gets the directional angle-delta distributions. + /// + public Av1Distribution[] AngleDelta { get; } + + /// + /// Gets the filter-intra mode distribution. + /// + public Av1Distribution FilterIntraMode { get; } + + /// + /// Gets the filter-intra enable distributions. + /// + public Av1Distribution[] FilterIntra { get; } + + /// + /// Gets the transform-size distributions. + /// + public Av1Distribution[][] TransformSize { get; } + + /// + /// Gets the end-of-block token distributions selected for the frame base quantizer. + /// + public Av1Distribution[][][] EndOfBlockFlag { get; } + + /// + /// Gets the coefficient base-level distributions selected for the frame base quantizer. + /// + public Av1Distribution[][][] CoefficientsBase { get; } + + /// + /// Gets the final-nonzero coefficient distributions selected for the frame base quantizer. + /// + public Av1Distribution[][][] BaseEndOfBlock { get; } + + /// + /// Gets the DC sign distributions selected for the frame base quantizer. + /// + public Av1Distribution[][] DcSign { get; } + + /// + /// Gets the coefficient base-range distributions selected for the frame base quantizer. + /// + public Av1Distribution[][][] CoefficientsBaseRange { get; } + + /// + /// Gets the transform-block skip distributions selected for the frame base quantizer. + /// + public Av1Distribution[][] TransformBlockSkip { get; } + + /// + /// Gets the end-of-block extra-bit distributions selected for the frame base quantizer. + /// + public Av1Distribution[][][] EndOfBlockExtra { get; } + + /// + /// Gets the joint chroma-from-luma sign distribution. + /// + public Av1Distribution ChromaFromLumaSign { get; } + + /// + /// Gets the chroma-from-luma alpha-magnitude distributions. + /// + public Av1Distribution[] ChromaFromLumaAlpha { get; } + + /// + /// Gets the intra transform-type distributions. + /// + public Av1Distribution[][][] IntraExtendedTransform { get; } + + /// + /// Gets the inter transform-type distributions. + /// + public Av1Distribution[][] InterExtendedTransform { get; } + + /// + /// Restores the normative frame defaults selected by a base quantizer index. + /// + /// The frame base quantizer index selecting coefficient distribution defaults. + public void ResetToDefaults(int qIndex) + { + int qContext = GetQContext(qIndex); + + // The prototypes are never exposed to a range reader. Copying their state lets a decoder session reuse the + // same three mutable object graphs even when successive frames select different coefficient-model bands. + this.CopyFrom(DefaultPrototypes[qContext]); + } + + /// + /// Maps a frame base quantizer to its normative coefficient-probability initialization band. + /// + /// The frame base quantizer index. + /// The zero-based quantizer-band index. + private static int GetQContext(int qIndex) + => qIndex switch + { + <= FirstQuantizerBandMaximum => 0, + <= SecondQuantizerBandMaximum => 1, + <= ThirdQuantizerBandMaximum => 2, + _ => 3 + }; + + /// + /// Replaces every probability threshold and adaptation count with state copied from another frame context. + /// + /// The frame context state to copy. + public void CopyFrom(Av1FrameEntropyContext source) + { + this.IntraBlockCopy.CopyFrom(source.IntraBlockCopy); + this.DisplacementVector.CopyFrom(source.DisplacementVector); + this.SwitchableRestoration.CopyFrom(source.SwitchableRestoration); + this.WienerRestoration.CopyFrom(source.WienerRestoration); + this.SgrProjectionRestoration.CopyFrom(source.SgrProjectionRestoration); + CopyState(source.PaletteYMode, this.PaletteYMode); + CopyState(source.PaletteUvMode, this.PaletteUvMode); + CopyState(source.PaletteYSize, this.PaletteYSize); + CopyState(source.PaletteUvSize, this.PaletteUvSize); + CopyState(source.PaletteYColorIndex, this.PaletteYColorIndex); + CopyState(source.PaletteUvColorIndex, this.PaletteUvColorIndex); + CopyState(source.PartitionTypes, this.PartitionTypes); + CopyState(source.KeyFrameYMode, this.KeyFrameYMode); + CopyState(source.UvMode, this.UvMode); + CopyState(source.Skip, this.Skip); + CopyState(source.SkipMode, this.SkipMode); + this.DeltaLoopFilterAbsolute.CopyFrom(source.DeltaLoopFilterAbsolute); + this.DeltaQuantizerAbsolute.CopyFrom(source.DeltaQuantizerAbsolute); + CopyState(source.SegmentId, this.SegmentId); + CopyState(source.AngleDelta, this.AngleDelta); + this.FilterIntraMode.CopyFrom(source.FilterIntraMode); + CopyState(source.FilterIntra, this.FilterIntra); + CopyState(source.TransformSize, this.TransformSize); + CopyState(source.EndOfBlockFlag, this.EndOfBlockFlag); + CopyState(source.CoefficientsBase, this.CoefficientsBase); + CopyState(source.BaseEndOfBlock, this.BaseEndOfBlock); + CopyState(source.DcSign, this.DcSign); + CopyState(source.CoefficientsBaseRange, this.CoefficientsBaseRange); + CopyState(source.TransformBlockSkip, this.TransformBlockSkip); + CopyState(source.EndOfBlockExtra, this.EndOfBlockExtra); + this.ChromaFromLumaSign.CopyFrom(source.ChromaFromLumaSign); + CopyState(source.ChromaFromLumaAlpha, this.ChromaFromLumaAlpha); + CopyState(source.IntraExtendedTransform, this.IntraExtendedTransform); + CopyState(source.InterExtendedTransform, this.InterExtendedTransform); + } + + /// + /// Copies this tile-adapted context into a destination used as completed frame state. + /// + /// The independently owned frame context that receives the snapshot. + /// + /// AV1 resets CDF observation counters after publishing the context-update tile. The copied thresholds remain + /// adapted, while the next frame starts its update-rate history from zero. + /// + public void SnapshotTo(Av1FrameEntropyContext destination) + { + destination.CopyFrom(this); + destination.ResetUpdateCounts(); + } + + /// + /// Resets the observation count of every distribution without changing its probability thresholds. + /// + private void ResetUpdateCounts() + { + this.IntraBlockCopy.ResetUpdateCount(); + this.DisplacementVector.ResetUpdateCounts(); + this.SwitchableRestoration.ResetUpdateCount(); + this.WienerRestoration.ResetUpdateCount(); + this.SgrProjectionRestoration.ResetUpdateCount(); + ResetUpdateCounts(this.PaletteYMode); + ResetUpdateCounts(this.PaletteUvMode); + ResetUpdateCounts(this.PaletteYSize); + ResetUpdateCounts(this.PaletteUvSize); + ResetUpdateCounts(this.PaletteYColorIndex); + ResetUpdateCounts(this.PaletteUvColorIndex); + ResetUpdateCounts(this.PartitionTypes); + ResetUpdateCounts(this.KeyFrameYMode); + ResetUpdateCounts(this.UvMode); + ResetUpdateCounts(this.Skip); + ResetUpdateCounts(this.SkipMode); + this.DeltaLoopFilterAbsolute.ResetUpdateCount(); + this.DeltaQuantizerAbsolute.ResetUpdateCount(); + ResetUpdateCounts(this.SegmentId); + ResetUpdateCounts(this.AngleDelta); + this.FilterIntraMode.ResetUpdateCount(); + ResetUpdateCounts(this.FilterIntra); + ResetUpdateCounts(this.TransformSize); + ResetUpdateCounts(this.EndOfBlockFlag); + ResetUpdateCounts(this.CoefficientsBase); + ResetUpdateCounts(this.BaseEndOfBlock); + ResetUpdateCounts(this.DcSign); + ResetUpdateCounts(this.CoefficientsBaseRange); + ResetUpdateCounts(this.TransformBlockSkip); + ResetUpdateCounts(this.EndOfBlockExtra); + this.ChromaFromLumaSign.ResetUpdateCount(); + ResetUpdateCounts(this.ChromaFromLumaAlpha); + ResetUpdateCounts(this.IntraExtendedTransform); + ResetUpdateCounts(this.InterExtendedTransform); + } + + /// + /// Copies one distribution row into an existing row with the same default-table shape. + /// + /// The source distribution row. + /// The destination distribution row. + private static void CopyState(Av1Distribution[] source, Av1Distribution[] destination) + { + for (int index = 0; index < source.Length; index++) + { + destination[index].CopyFrom(source[index]); + } + } + + /// + /// Copies a two-dimensional distribution table into an existing table with the same default-table shape. + /// + /// The source distribution table. + /// The destination distribution table. + private static void CopyState(Av1Distribution[][] source, Av1Distribution[][] destination) + { + for (int index = 0; index < source.Length; index++) + { + CopyState(source[index], destination[index]); + } + } + + /// + /// Copies a three-dimensional distribution table into an existing table with the same default-table shape. + /// + /// The source distribution table. + /// The destination distribution table. + private static void CopyState(Av1Distribution[][][] source, Av1Distribution[][][] destination) + { + for (int index = 0; index < source.Length; index++) + { + CopyState(source[index], destination[index]); + } + } + + /// + /// Resets observation counts in one distribution row. + /// + /// The distribution row to reset. + private static void ResetUpdateCounts(Av1Distribution[] distributions) + { + for (int index = 0; index < distributions.Length; index++) + { + distributions[index].ResetUpdateCount(); + } + } + + /// + /// Resets observation counts in a two-dimensional distribution table. + /// + /// The distribution table to reset. + private static void ResetUpdateCounts(Av1Distribution[][] distributions) + { + for (int index = 0; index < distributions.Length; index++) + { + ResetUpdateCounts(distributions[index]); + } + } + + /// + /// Resets observation counts in a three-dimensional distribution table. + /// + /// The distribution table to reset. + private static void ResetUpdateCounts(Av1Distribution[][][] distributions) + { + for (int index = 0; index < distributions.Length; index++) + { + ResetUpdateCounts(distributions[index]); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContexts.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContexts.cs new file mode 100644 index 000000000..476218527 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContexts.cs @@ -0,0 +1,150 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; + +/// +/// Owns the reusable frame-base, tile-working, and published AV1 entropy contexts for one decoder session. +/// +internal sealed class Av1FrameEntropyContexts +{ + /// + /// The maximum number of live reference-map and presentation owners plus the newly reconstructed frame awaiting + /// commit. + /// + private const int MaximumSnapshotCount = Av1Constants.ReferenceFrameCount + 2; + + /// + /// Session-local returned snapshot graphs available for later refreshed frames. + /// + private InlineArray10 returnedSnapshots; + + /// + /// The number of returned snapshot graphs currently available for reuse. + /// + private int returnedSnapshotCount; + + /// + /// The base quantizer index used to initialize a newly required snapshot graph. + /// + private int currentQIndex; + + /// + /// Initializes a new instance of the class. + /// + /// The initial frame base quantizer index. + public Av1FrameEntropyContexts(int qIndex) + { + this.Base = new(qIndex); + this.Working = new(qIndex); + this.Published = new(qIndex); + this.currentQIndex = qIndex; + } + + /// + /// Gets the unchanged frame context from which each independently decoded tile starts. + /// + public Av1FrameEntropyContext Base { get; } + + /// + /// Gets the tile-local context reused sequentially for each tile in the current frame. + /// + public Av1FrameEntropyContext Working { get; } + + /// + /// Gets the completed frame context selected by the signaled context-update tile, or the unchanged frame-base + /// context when frame-end updates are disabled. + /// + public Av1FrameEntropyContext Published { get; } + + /// + /// Initializes frame entropy state from either a retained primary reference or normative quantizer-band defaults. + /// + /// The frame base quantizer index selecting coefficient distribution defaults. + /// + /// The retained primary-reference context, or when the frame selects normative defaults. + /// + public void BeginFrame(int qIndex, Av1FrameEntropyContext? primaryReferenceContext) + { + this.currentQIndex = qIndex; + if (primaryReferenceContext is null) + { + this.Base.ResetToDefaults(qIndex); + } + else + { + // A retained context is independent from the working and published graphs. Copying it here preserves the + // reference owner's snapshot while the current frame adapts its own tile-local state. + this.Base.CopyFrom(primaryReferenceContext); + } + + // The context-update tile can precede later tiles. Published therefore cannot alias Working: a later tile + // must be free to overwrite Working while the selected completed-frame state remains available to the owner. + this.Base.SnapshotTo(this.Published); + } + + /// + /// Clears active frame entropy state when a new coded sequence invalidates the complete reference map. + /// + public void Reset() + { + this.currentQIndex = 0; + this.Base.ResetToDefaults(this.currentQIndex); + this.Base.SnapshotTo(this.Working); + this.Base.SnapshotTo(this.Published); + + // Returned graphs contain no live reference state and remain private to this decoder. Retaining them here + // allows the next sequence to reuse peak reference ownership without a static cross-decode pool. + } + + /// + /// Rents an independently owned, reset-counter snapshot of the completed frame entropy context. + /// + /// The snapshot that must later be returned through . + public Av1FrameEntropyContext RentPublishedSnapshot() + { + Av1FrameEntropyContext snapshot; + if (this.returnedSnapshotCount == 0) + { + // Eight slots can own distinct frames while the selected output owns a ninth frame no longer present in + // the map. Rent one further graph before commit releases the owner displaced by the completed frame. + snapshot = new(this.currentQIndex); + } + else + { + int snapshotIndex = --this.returnedSnapshotCount; + snapshot = this.returnedSnapshots[snapshotIndex]!; + this.returnedSnapshots[snapshotIndex] = null; + } + + this.Published.SnapshotTo(snapshot); + return snapshot; + } + + /// + /// Returns a retained-frame entropy snapshot to this decoder session for later reuse. + /// + /// The snapshot whose reference-frame ownership has ended. + public void ReturnSnapshot(Av1FrameEntropyContext snapshot) + { + // The fixed capacity covers eight distinct slot owners, one detached presentation owner, and the replacement + // frame rented before commit. Av1ReferenceFrame returns each graph exactly once, so the session cannot exceed + // this bound. + this.returnedSnapshots[this.returnedSnapshotCount++] = snapshot; + } + + /// + /// Provides inline storage for every entropy snapshot graph that one decoder session can allocate concurrently. + /// + /// The stored reference type. + [InlineArray(MaximumSnapshotCount)] + private struct InlineArray10 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs index 3af2b11a0..b61ccd038 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs @@ -439,11 +439,9 @@ internal static class Av1SymbolContextHelper /// The transform type associated with the selected prediction mode. internal static Av1TransformType ConvertIntraModeToTransformType(Av1BlockModeInfo modeInfo, Av1PlaneType planeType) { - Av1PredictionMode mode = (planeType == Av1PlaneType.Y) ? modeInfo.YMode : modeInfo.UvMode; - if (mode == Av1PredictionMode.UvChromaFromLuma) - { - mode = Av1PredictionMode.DC; - } + // libaom's get_uv_mode() is the explicit boundary between the distinct UV and luma prediction domains. CfL maps + // to DC because the chroma AC contribution is applied to a DC predictor before coefficient reconstruction. + Av1PredictionMode mode = planeType == Av1PlaneType.Y ? modeInfo.YMode : modeInfo.UvMode.ToLumaMode(); return mode.ToTransformType(); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolDecoder.cs index 8aa979c56..308ca9751 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolDecoder.cs @@ -15,179 +15,9 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; internal ref struct Av1SymbolDecoder { /// - /// Maps each intra prediction mode to the reduced neighbor context used by key-frame luma modes. + /// The independently adaptable distribution graph for the current tile. /// - private static readonly int[] IntraModeContext = [0, 1, 2, 3, 4, 4, 4, 4, 3, 0, 1, 2, 0]; - - /// - /// The tile-adaptive intra-block-copy distribution. - /// - private readonly Av1Distribution tileIntraBlockCopy; - - /// - /// The tile-adaptive integer displacement-vector context used by intra-block copy. - /// - private readonly Av1DisplacementVectorContext displacementVector; - - /// - /// The tile-adaptive switchable loop-restoration distribution. - /// - private readonly Av1Distribution switchableRestoration; - - /// - /// The tile-adaptive Wiener loop-restoration distribution. - /// - private readonly Av1Distribution wienerRestoration; - - /// - /// The tile-adaptive self-guided loop-restoration distribution. - /// - private readonly Av1Distribution sgrProjectionRestoration; - - /// - /// The tile-adaptive luma palette-mode distributions. - /// - private readonly Av1Distribution[][] paletteYMode; - - /// - /// The tile-adaptive chroma palette-mode distributions. - /// - private readonly Av1Distribution[] paletteUvMode; - - /// - /// The tile-adaptive luma palette-size distributions. - /// - private readonly Av1Distribution[] paletteYSize; - - /// - /// The tile-adaptive chroma palette-size distributions. - /// - private readonly Av1Distribution[] paletteUvSize; - - /// - /// The tile-adaptive luma palette color-index distributions. - /// - private readonly Av1Distribution[][] paletteYColorIndex; - - /// - /// The tile-adaptive chroma palette color-index distributions. - /// - private readonly Av1Distribution[][] paletteUvColorIndex; - - /// - /// The tile-adaptive partition-type distributions. - /// - private readonly Av1Distribution[] tilePartitionTypes; - - /// - /// The tile-adaptive key-frame luma-mode distributions. - /// - private readonly Av1Distribution[][] keyFrameYMode; - - /// - /// The tile-adaptive chroma intra-mode distributions. - /// - private readonly Av1Distribution[][] uvMode; - - /// - /// The tile-adaptive transform-skip distributions. - /// - private readonly Av1Distribution[] skip; - - /// - /// The tile-adaptive skip-mode distributions. - /// - private readonly Av1Distribution[] skipMode; - - /// - /// The tile-adaptive absolute loop-filter delta distribution. - /// - private readonly Av1Distribution deltaLoopFilterAbsolute; - - /// - /// The tile-adaptive absolute quantizer delta distribution. - /// - private readonly Av1Distribution deltaQuantizerAbsolute; - - /// - /// The tile-adaptive spatial segment-identifier distributions. - /// - private readonly Av1Distribution[] segmentId; - - /// - /// The tile-adaptive directional angle-delta distributions. - /// - private readonly Av1Distribution[] angleDelta; - - /// - /// The tile-adaptive filter-intra mode distribution. - /// - private readonly Av1Distribution filterIntraMode; - - /// - /// The tile-adaptive filter-intra enable distributions. - /// - private readonly Av1Distribution[] filterIntra; - - /// - /// The tile-adaptive transform-size distributions. - /// - private readonly Av1Distribution[][] transformSize; - - /// - /// The tile-adaptive end-of-block token distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][][] endOfBlockFlag; - - /// - /// The tile-adaptive coefficient base-level distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][][] coefficientsBase; - - /// - /// The tile-adaptive final-nonzero coefficient distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][][] baseEndOfBlock; - - /// - /// The tile-adaptive DC sign distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][] dcSign; - - /// - /// The tile-adaptive coefficient base-range distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][][] coefficientsBaseRange; - - /// - /// The tile-adaptive transform-block skip distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][] transformBlockSkip; - - /// - /// The tile-adaptive end-of-block extra-bit distributions selected for the frame base quantizer. - /// - private readonly Av1Distribution[][][] endOfBlockExtra; - - /// - /// The tile-adaptive joint chroma-from-luma sign distribution. - /// - private readonly Av1Distribution chromaFromLumaSign; - - /// - /// The tile-adaptive chroma-from-luma alpha-magnitude distributions. - /// - private readonly Av1Distribution[] chromaFromLumaAlpha; - - /// - /// The tile-adaptive intra transform-type distributions. - /// - private readonly Av1Distribution[][][] intraExtendedTransform; - - /// - /// The tile-adaptive inter transform-type distributions. - /// - private readonly Av1Distribution[][] interExtendedTransform; + private readonly Av1FrameEntropyContext context; /// /// The configuration providing temporary coefficient-context memory. @@ -207,47 +37,42 @@ internal ref struct Av1SymbolDecoder /// The frame base quantizer index. /// A value indicating whether decoded symbols adapt their tile distributions. public Av1SymbolDecoder(Configuration configuration, Span tileData, int qIndex, bool updateCdf = true) + : this(configuration, tileData, new Av1FrameEntropyContext(qIndex), updateCdf) { - // Every tile starts from its own frame-context copy. Sharing these objects would let one image's adaptive - // updates change the initial probabilities used to decode the next tile or image. - this.tileIntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy.CreateCopy(); - this.displacementVector = new(); - this.switchableRestoration = Av1DefaultDistributions.SwitchableRestoration.CreateCopy(); - this.wienerRestoration = Av1DefaultDistributions.WienerRestoration.CreateCopy(); - this.sgrProjectionRestoration = Av1DefaultDistributions.SgrProjectionRestoration.CreateCopy(); - this.paletteYMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.PaletteYMode); - this.paletteUvMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.PaletteUvMode); - this.paletteYSize = Av1Distribution.CreateCopy(Av1DefaultDistributions.PaletteYSize); - this.paletteUvSize = Av1Distribution.CreateCopy(Av1DefaultDistributions.PaletteUvSize); - this.paletteYColorIndex = Av1Distribution.CreateCopy(Av1DefaultDistributions.PaletteYColorIndex); - this.paletteUvColorIndex = Av1Distribution.CreateCopy(Av1DefaultDistributions.PaletteUvColorIndex); - this.tilePartitionTypes = Av1Distribution.CreateCopy(Av1DefaultDistributions.PartitionTypes); - this.keyFrameYMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.KeyFrameYMode); - this.uvMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.UvMode); - this.skip = Av1Distribution.CreateCopy(Av1DefaultDistributions.Skip); - this.skipMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.SkipMode); - this.deltaLoopFilterAbsolute = Av1DefaultDistributions.DeltaLoopFilterAbsolute.CreateCopy(); - this.deltaQuantizerAbsolute = Av1DefaultDistributions.DeltaQuantizerAbsolute.CreateCopy(); - this.segmentId = Av1Distribution.CreateCopy(Av1DefaultDistributions.SegmentId); - this.angleDelta = Av1Distribution.CreateCopy(Av1DefaultDistributions.AngleDelta); - this.filterIntraMode = Av1DefaultDistributions.FilterIntraMode.CreateCopy(); - this.filterIntra = Av1Distribution.CreateCopy(Av1DefaultDistributions.FilterIntra); - this.transformSize = Av1Distribution.CreateCopy(Av1DefaultDistributions.TransformSize); - this.chromaFromLumaSign = Av1DefaultDistributions.ChromaFromLumaSign.CreateCopy(); - this.chromaFromLumaAlpha = Av1Distribution.CreateCopy(Av1DefaultDistributions.ChromaFromLumaAlpha); - this.intraExtendedTransform = Av1Distribution.CreateCopy(Av1DefaultDistributions.IntraExtendedTransform); - this.interExtendedTransform = Av1Distribution.CreateCopy(Av1DefaultDistributions.InterExtendedTransform); + } + + /// + /// Initializes a new instance of the struct over a caller-owned tile entropy + /// context. + /// + /// The configuration providing temporary memory. + /// The entropy-coded tile payload. + /// The independently adaptable context initialized for this tile. + /// A value indicating whether decoded symbols adapt their tile distributions. + public Av1SymbolDecoder( + Configuration configuration, + Span tileData, + Av1FrameEntropyContext context, + bool updateCdf) + { + // The context owner controls reset and publication. Holding one reference here keeps the range decoder small + // and prevents a second set of aliases from becoming a competing source of entropy state. + this.context = context; this.configuration = configuration; this.reader = new Av1SymbolReader(tileData, updateCdf); - this.endOfBlockFlag = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetEndOfBlockFlag(qIndex)); - this.coefficientsBase = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetCoefficientsBase(qIndex)); - this.baseEndOfBlock = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetBaseEndOfBlock(qIndex)); - this.dcSign = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetDcSign(qIndex)); - this.coefficientsBaseRange = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetCoefficientsBaseRange(qIndex)); - this.transformBlockSkip = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetTransformBlockSkip(qIndex)); - this.endOfBlockExtra = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetEndOfBlockExtra(qIndex)); } + /// + /// Gets the reduced neighbor context for each intra prediction mode used by key-frame luma modes. + /// + private static ReadOnlySpan IntraModeContext => [0, 1, 2, 3, 4, 4, 4, 4, 3, 0, 1, 2, 0]; + + /// + /// Validates that range decoding remained within the bounded tile payload and consumed the required trailing-one bit. + /// + public void ValidateTrailingBits() + => this.reader.ValidateTrailingBits(); + /// /// Reads a fixed-width CDEF strength index. /// @@ -315,7 +140,7 @@ internal ref struct Av1SymbolDecoder public Av1RestorationFilterType ReadSwitchableRestorationType() { ref Av1SymbolReader r = ref this.reader; - return (Av1RestorationFilterType)r.ReadSymbol(this.switchableRestoration); + return (Av1RestorationFilterType)r.ReadSymbol(this.context.SwitchableRestoration); } /// @@ -325,7 +150,7 @@ internal ref struct Av1SymbolDecoder public bool ReadWienerRestoration() { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.wienerRestoration) != 0; + return r.ReadSymbol(this.context.WienerRestoration) != 0; } /// @@ -335,7 +160,7 @@ internal ref struct Av1SymbolDecoder public bool ReadSgrProjectionRestoration() { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.sgrProjectionRestoration) != 0; + return r.ReadSymbol(this.context.SgrProjectionRestoration) != 0; } /// @@ -398,7 +223,7 @@ internal ref struct Av1SymbolDecoder public bool ReadPaletteYMode(int blockSizeContext, int neighborContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.paletteYMode[blockSizeContext][neighborContext]) != 0; + return r.ReadSymbol(this.context.PaletteYMode[blockSizeContext][neighborContext]) != 0; } /// @@ -409,7 +234,7 @@ internal ref struct Av1SymbolDecoder public bool ReadPaletteUvMode(bool hasLumaPalette) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.paletteUvMode[hasLumaPalette ? 1 : 0]) != 0; + return r.ReadSymbol(this.context.PaletteUvMode[hasLumaPalette ? 1 : 0]) != 0; } /// @@ -422,8 +247,8 @@ internal ref struct Av1SymbolDecoder { ref Av1SymbolReader r = ref this.reader; Av1Distribution distribution = planeType == Av1PlaneType.Y - ? this.paletteYSize[blockSizeContext] - : this.paletteUvSize[blockSizeContext]; + ? this.context.PaletteYSize[blockSizeContext] + : this.context.PaletteUvSize[blockSizeContext]; return r.ReadSymbol(distribution) + 2; } @@ -439,8 +264,8 @@ internal ref struct Av1SymbolDecoder { ref Av1SymbolReader r = ref this.reader; Av1Distribution distribution = planeType == Av1PlaneType.Y - ? this.paletteYColorIndex[paletteSize - 2][colorContext] - : this.paletteUvColorIndex[paletteSize - 2][colorContext]; + ? this.context.PaletteYColorIndex[paletteSize - 2][colorContext] + : this.context.PaletteUvColorIndex[paletteSize - 2][colorContext]; return r.ReadSymbol(distribution); } @@ -452,7 +277,7 @@ internal ref struct Av1SymbolDecoder public bool ReadUseIntraBlockCopy() { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.tileIntraBlockCopy) > 0; + return r.ReadSymbol(this.context.IntraBlockCopy) > 0; } /// @@ -461,7 +286,7 @@ internal ref struct Av1SymbolDecoder /// The spatially derived reference vector. /// The decoded displacement vector in one-eighth-sample units. public Av1MotionVector ReadDisplacementVector(Av1MotionVector reference) - => this.displacementVector.Read(ref this.reader, reference); + => this.context.DisplacementVector.Read(ref this.reader, reference); /// /// Reads a complete block partition type from the selected partition context. @@ -471,7 +296,7 @@ internal ref struct Av1SymbolDecoder public Av1PartitionType ReadPartitionType(int context) { ref Av1SymbolReader r = ref this.reader; - return (Av1PartitionType)r.ReadSymbol(this.tilePartitionTypes[context]); + return (Av1PartitionType)r.ReadSymbol(this.context.PartitionTypes[context]); } /// @@ -482,7 +307,7 @@ internal ref struct Av1SymbolDecoder /// or . public Av1PartitionType ReadSplitOrHorizontal(Av1BlockSize blockSize, int context) { - uint frequency = GetSplitOrHorizontalFrequency(this.tilePartitionTypes, blockSize, context); + uint frequency = GetSplitOrHorizontalFrequency(this.context.PartitionTypes, blockSize, context); ref Av1SymbolReader r = ref this.reader; return r.ReadBoolean(frequency) ? Av1PartitionType.Split : Av1PartitionType.Horizontal; } @@ -495,7 +320,7 @@ internal ref struct Av1SymbolDecoder /// or . public Av1PartitionType ReadSplitOrVertical(Av1BlockSize blockSize, int context) { - uint frequency = GetSplitOrVerticalFrequency(this.tilePartitionTypes, blockSize, context); + uint frequency = GetSplitOrVerticalFrequency(this.context.PartitionTypes, blockSize, context); ref Av1SymbolReader r = ref this.reader; return r.ReadBoolean(frequency) ? Av1PartitionType.Split : Av1PartitionType.Vertical; } @@ -523,7 +348,7 @@ internal ref struct Av1SymbolDecoder int aboveContext = IntraModeContext[(int)aboveMode]; int leftContext = IntraModeContext[(int)leftMode]; - return (Av1PredictionMode)r.ReadSymbol(this.keyFrameYMode[aboveContext][leftContext]); + return (Av1PredictionMode)r.ReadSymbol(this.context.KeyFrameYMode[aboveContext][leftContext]); } /// @@ -532,11 +357,11 @@ internal ref struct Av1SymbolDecoder /// The decoded luma prediction mode. /// Indicates whether chroma-from-luma is valid for the block. /// The decoded chroma prediction mode. - public Av1PredictionMode ReadIntraModeUv(Av1PredictionMode mode, bool chromaFromLumaAllowed) + public Av1ChromaPredictionMode ReadIntraModeUv(Av1PredictionMode mode, bool chromaFromLumaAllowed) { int chromaForLumaIndex = chromaFromLumaAllowed ? 1 : 0; ref Av1SymbolReader r = ref this.reader; - return (Av1PredictionMode)r.ReadSymbol(this.uvMode[chromaForLumaIndex][(int)mode]); + return (Av1ChromaPredictionMode)r.ReadSymbol(this.context.UvMode[chromaForLumaIndex][(int)mode]); } /// @@ -547,7 +372,7 @@ internal ref struct Av1SymbolDecoder public bool ReadSkip(int ctx) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.skip[ctx]) > 0; + return r.ReadSymbol(this.context.Skip[ctx]) > 0; } /// @@ -558,7 +383,7 @@ internal ref struct Av1SymbolDecoder public bool ReadSkipMode(int context) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.skipMode[context]) > 0; + return r.ReadSymbol(this.context.SkipMode[context]) > 0; } /// @@ -568,7 +393,7 @@ internal ref struct Av1SymbolDecoder public int ReadDeltaLoopFilter() { ref Av1SymbolReader r = ref this.reader; - int deltaLoopFilterAbsolute = r.ReadSymbol(this.deltaLoopFilterAbsolute); + int deltaLoopFilterAbsolute = r.ReadSymbol(this.context.DeltaLoopFilterAbsolute); if (deltaLoopFilterAbsolute == Av1Constants.DeltaLoopFilterSmall) { int deltaLoopFilterRemainingBits = r.ReadLiteral(3) + 1; @@ -592,7 +417,7 @@ internal ref struct Av1SymbolDecoder public int ReadDeltaQuantizerIndex() { ref Av1SymbolReader r = ref this.reader; - int deltaQuantizerAbsolute = r.ReadSymbol(this.deltaQuantizerAbsolute); + int deltaQuantizerAbsolute = r.ReadSymbol(this.context.DeltaQuantizerAbsolute); if (deltaQuantizerAbsolute == Av1Constants.DeltaQuantizerSmall) { int deltaQuantizerRemainingBits = r.ReadLiteral(3) + 1; @@ -617,7 +442,7 @@ internal ref struct Av1SymbolDecoder public int ReadSegmentId(int context) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.segmentId[context]); + return r.ReadSymbol(this.context.SegmentId[context]); } /// @@ -628,7 +453,7 @@ internal ref struct Av1SymbolDecoder public int ReadAngleDelta(Av1PredictionMode mode) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.angleDelta[(int)mode - 1]); + return r.ReadSymbol(this.context.AngleDelta[(int)mode - 1]); } /// @@ -640,10 +465,10 @@ internal ref struct Av1SymbolDecoder { ref Av1SymbolReader r = ref this.reader; Av1FilterIntraMode filterIntraMode = Av1FilterIntraMode.AllFilterIntraModes; - bool useFilterIntra = r.ReadSymbol(this.filterIntra[(int)blockSize]) > 0; + bool useFilterIntra = r.ReadSymbol(this.context.FilterIntra[(int)blockSize]) > 0; if (useFilterIntra) { - filterIntraMode = (Av1FilterIntraMode)r.ReadSymbol(this.filterIntraMode); + filterIntraMode = (Av1FilterIntraMode)r.ReadSymbol(this.context.FilterIntraMode); } return filterIntraMode; @@ -669,7 +494,7 @@ internal ref struct Av1SymbolDecoder DebugGuard.MustBeLessThanOrEqualTo(depth, Av1Constants.MaxTransformCategories, nameof(depth)); int category = depth - 1; - int value = r.ReadSymbol(this.transformSize[category][context]); + int value = r.ReadSymbol(this.context.TransformSize[category][context]); Av1TransformSize transformSize = blockSize.GetMaximumTransformSize(); for (int d = 0; d < value; ++d) { @@ -716,7 +541,7 @@ internal ref struct Av1SymbolDecoder int symbol; if (isInter) { - symbol = r.ReadSymbol(this.interExtendedTransform[extendedSet][(int)squareTransformSize]); + symbol = r.ReadSymbol(this.context.InterExtendedTransform[extendedSet][(int)squareTransformSize]); } else { @@ -724,7 +549,7 @@ internal ref struct Av1SymbolDecoder ? filterIntraMode.ToIntraDirection() : intraDirection; - symbol = r.ReadSymbol(this.intraExtendedTransform[extendedSet][(int)squareTransformSize][(int)intraMode]); + symbol = r.ReadSymbol(this.context.IntraExtendedTransform[extendedSet][(int)squareTransformSize][(int)intraMode]); } transformType = Av1SymbolContextHelper.GetExtendedTransformType(transformSetType, symbol); @@ -742,7 +567,7 @@ internal ref struct Av1SymbolDecoder public bool ReadTransformBlockSkip(Av1TransformSize transformSizeContext, int skipContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.transformBlockSkip[(int)transformSizeContext][skipContext]) > 0; + return r.ReadSymbol(this.context.TransformBlockSkip[(int)transformSizeContext][skipContext]) > 0; } /// @@ -752,7 +577,7 @@ internal ref struct Av1SymbolDecoder public int ReadChromFromLumaSign() { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.chromaFromLumaSign); + return r.ReadSymbol(this.context.ChromaFromLumaSign); } /// @@ -764,7 +589,7 @@ internal ref struct Av1SymbolDecoder { ref Av1SymbolReader r = ref this.reader; int context = Av1ChromaFromLumaMath.ContextU(jointSignPlus1 - 1); - return r.ReadSymbol(this.chromaFromLumaAlpha[context]); + return r.ReadSymbol(this.context.ChromaFromLumaAlpha[context]); } /// @@ -776,7 +601,7 @@ internal ref struct Av1SymbolDecoder { ref Av1SymbolReader r = ref this.reader; int context = Av1ChromaFromLumaMath.ContextV(jointSignPlus1 - 1); - return r.ReadSymbol(this.chromaFromLumaAlpha[context]); + return r.ReadSymbol(this.context.ChromaFromLumaAlpha[context]); } /// @@ -1082,7 +907,7 @@ internal ref struct Av1SymbolDecoder int endOfBlockContext = transformClass == Av1TransformClass.Class2D ? 0 : 1; int endOfBlockMultiSize = transformSize.GetLog2Minus4(); ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.endOfBlockFlag[endOfBlockMultiSize][(int)planeType][endOfBlockContext]) + 1; + return r.ReadSymbol(this.context.EndOfBlockFlag[endOfBlockMultiSize][(int)planeType][endOfBlockContext]) + 1; } /// @@ -1095,7 +920,7 @@ internal ref struct Av1SymbolDecoder private bool ReadEndOfBlockExtra(Av1TransformSize transformSizeContext, Av1PlaneType planeType, int endOfBlockContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.endOfBlockExtra[(int)transformSizeContext][(int)planeType][endOfBlockContext]) > 0; + return r.ReadSymbol(this.context.EndOfBlockExtra[(int)transformSizeContext][(int)planeType][endOfBlockContext]) > 0; } /// @@ -1108,7 +933,7 @@ internal ref struct Av1SymbolDecoder private int ReadCoefficientsBaseRange(Av1TransformSize transformSizeContext, Av1PlaneType planeType, int baseRangeContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.coefficientsBaseRange[(int)transformSizeContext][(int)planeType][baseRangeContext]); + return r.ReadSymbol(this.context.CoefficientsBaseRange[(int)transformSizeContext][(int)planeType][baseRangeContext]); } /// @@ -1120,7 +945,7 @@ internal ref struct Av1SymbolDecoder private int ReadDcSign(Av1PlaneType planeType, int dcSignContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.dcSign[(int)planeType][dcSignContext]); + return r.ReadSymbol(this.context.DcSign[(int)planeType][dcSignContext]); } /// @@ -1133,7 +958,7 @@ internal ref struct Av1SymbolDecoder private int ReadBaseEndOfBlock(Av1TransformSize transformSizeContext, Av1PlaneType planeType, int coefficientContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.baseEndOfBlock[(int)transformSizeContext][(int)planeType][coefficientContext]); + return r.ReadSymbol(this.context.BaseEndOfBlock[(int)transformSizeContext][(int)planeType][coefficientContext]); } /// @@ -1146,7 +971,7 @@ internal ref struct Av1SymbolDecoder private int ReadCoefficientsBase(Av1TransformSize transformSizeContext, Av1PlaneType planeType, int coefficientContext) { ref Av1SymbolReader r = ref this.reader; - return r.ReadSymbol(this.coefficientsBase[(int)transformSizeContext][(int)planeType][coefficientContext]); + return r.ReadSymbol(this.context.CoefficientsBase[(int)transformSizeContext][(int)planeType][coefficientContext]); } /// @@ -1160,7 +985,7 @@ internal ref struct Av1SymbolDecoder { ref Av1SymbolReader r = ref this.reader; Av1TransformSize limitedTransformSizeContext = (Av1TransformSize)Math.Min((int)transformSizeContext, (int)Av1TransformSize.Size32x32); - Av1Distribution distribution = this.coefficientsBaseRange[(int)limitedTransformSizeContext][(int)planeType][baseRangeContext]; + Av1Distribution distribution = this.context.CoefficientsBaseRange[(int)limitedTransformSizeContext][(int)planeType][baseRangeContext]; for (int idx = 0; idx < Av1Constants.CoefficientBaseRange; idx += Av1Constants.BaseRangeSizeMinus1) { int coefficientBaseRange = r.ReadSymbol(distribution); @@ -1177,7 +1002,7 @@ internal ref struct Av1SymbolDecoder /// /// The decoded nonnegative suffix value. /// The unary prefix exceeds the AV1 coefficient limit. - internal int ReadGolomb() + public int ReadGolomb() { ref Av1SymbolReader r = ref this.reader; int x = 1; @@ -1320,7 +1145,7 @@ internal ref struct Av1SymbolDecoder /// The current block size. /// The partition probability context. /// The Q15 probability of the split outcome. - internal static uint GetSplitOrHorizontalFrequency(Av1Distribution[] inputs, Av1BlockSize blockSize, int context) + public static uint GetSplitOrHorizontalFrequency(Av1Distribution[] inputs, Av1BlockSize blockSize, int context) { Av1Distribution input = inputs[context]; @@ -1346,7 +1171,7 @@ internal ref struct Av1SymbolDecoder /// The current block size. /// The partition probability context. /// The Q15 probability of the split outcome. - internal static uint GetSplitOrVerticalFrequency(Av1Distribution[] inputs, Av1BlockSize blockSize, int context) + public static uint GetSplitOrVerticalFrequency(Av1Distribution[] inputs, Av1BlockSize blockSize, int context) { Av1Distribution input = inputs[context]; diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs index e2b8e14c0..aa1fe9cc7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs @@ -155,29 +155,29 @@ internal class Av1SymbolEncoder : IDisposable /// A value indicating whether encoded symbols adapt their tile distributions. public Av1SymbolEncoder(Configuration configuration, int initialSize, int qIndex, bool updateCdf = true) { - // Encoding and decoding must begin from equivalent tile-local models. Copying the defaults also prevents - // one encoded image from changing the probabilities used by later encoder or decoder instances. - this.tileIntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy.CreateCopy(); - this.tilePartitionTypes = Av1Distribution.CreateCopy(Av1DefaultDistributions.PartitionTypes); - this.keyFrameYMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.KeyFrameYMode); - this.uvMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.UvMode); - this.filterIntra = Av1Distribution.CreateCopy(Av1DefaultDistributions.FilterIntra); - this.filterIntraMode = Av1DefaultDistributions.FilterIntraMode.CreateCopy(); - this.deltaQuantizerAbsolute = Av1DefaultDistributions.DeltaQuantizerAbsolute.CreateCopy(); - this.intraExtendedTransform = Av1Distribution.CreateCopy(Av1DefaultDistributions.IntraExtendedTransform); - this.segmentId = Av1Distribution.CreateCopy(Av1DefaultDistributions.SegmentId); - this.angleDelta = Av1Distribution.CreateCopy(Av1DefaultDistributions.AngleDelta); - this.skip = Av1Distribution.CreateCopy(Av1DefaultDistributions.Skip); - this.skipMode = Av1Distribution.CreateCopy(Av1DefaultDistributions.SkipMode); - this.chromaFromLumaSign = Av1DefaultDistributions.ChromaFromLumaSign.CreateCopy(); - this.chromaFromLumaAlpha = Av1Distribution.CreateCopy(Av1DefaultDistributions.ChromaFromLumaAlpha); - this.transformBlockSkip = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetTransformBlockSkip(qIndex)); - this.endOfBlockFlag = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetEndOfBlockFlag(qIndex)); - this.coefficientsBaseRange = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetCoefficientsBaseRange(qIndex)); - this.coefficientsBase = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetCoefficientsBase(qIndex)); - this.coefficientsBaseEndOfBlock = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetBaseEndOfBlock(qIndex)); - this.dcSign = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetDcSign(qIndex)); - this.endOfBlockExtra = Av1Distribution.CreateCopy(Av1DefaultDistributions.GetEndOfBlockExtra(qIndex)); + // Every default accessor creates independently mutable state. Encoding and decoding therefore begin from + // equivalent tile-local models without constructing and immediately deep-copying a second object graph. + this.tileIntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy; + this.tilePartitionTypes = Av1DefaultDistributions.PartitionTypes; + this.keyFrameYMode = Av1DefaultDistributions.KeyFrameYMode; + this.uvMode = Av1DefaultDistributions.UvMode; + this.filterIntra = Av1DefaultDistributions.FilterIntra; + this.filterIntraMode = Av1DefaultDistributions.FilterIntraMode; + this.deltaQuantizerAbsolute = Av1DefaultDistributions.DeltaQuantizerAbsolute; + this.intraExtendedTransform = Av1DefaultDistributions.IntraExtendedTransform; + this.segmentId = Av1DefaultDistributions.SegmentId; + this.angleDelta = Av1DefaultDistributions.AngleDelta; + this.skip = Av1DefaultDistributions.Skip; + this.skipMode = Av1DefaultDistributions.SkipMode; + this.chromaFromLumaSign = Av1DefaultDistributions.ChromaFromLumaSign; + this.chromaFromLumaAlpha = Av1DefaultDistributions.ChromaFromLumaAlpha; + this.transformBlockSkip = Av1DefaultDistributions.GetTransformBlockSkip(qIndex); + this.endOfBlockFlag = Av1DefaultDistributions.GetEndOfBlockFlag(qIndex); + this.coefficientsBaseRange = Av1DefaultDistributions.GetCoefficientsBaseRange(qIndex); + this.coefficientsBase = Av1DefaultDistributions.GetCoefficientsBase(qIndex); + this.coefficientsBaseEndOfBlock = Av1DefaultDistributions.GetBaseEndOfBlock(qIndex); + this.dcSign = Av1DefaultDistributions.GetDcSign(qIndex); + this.endOfBlockExtra = Av1DefaultDistributions.GetEndOfBlockExtra(qIndex); this.configuration = configuration; this.writer = new(configuration, initialSize, updateCdf); this.baseQIndex = qIndex; @@ -375,7 +375,7 @@ internal class Av1SymbolEncoder : IDisposable /// The transform direction class. /// The signaled transform size selecting the token alphabet. /// The square transform-size probability context. - internal void WriteEndOfBlockPosition(ushort endOfBlock, Av1ComponentType componentType, Av1TransformClass transformClass, Av1TransformSize transformSize, Av1TransformSize transformSizeContext) + public void WriteEndOfBlockPosition(ushort endOfBlock, Av1ComponentType componentType, Av1TransformClass transformClass, Av1TransformSize transformSize, Av1TransformSize transformSizeContext) { short endOfBlockPosition = Av1SymbolContextHelper.GetEndOfBlockPosition(endOfBlock, out int eobExtra); this.WriteEndOfBlockFlag(componentType, transformClass, transformSize, endOfBlockPosition); @@ -406,7 +406,7 @@ internal class Av1SymbolEncoder : IDisposable /// Indicates whether the transform block is empty. /// The square transform-size probability context. /// The context derived from neighboring coefficient blocks. - internal void WriteTransformBlockSkip(bool skip, Av1TransformSize transformSizeContext, int skipContext) + public void WriteTransformBlockSkip(bool skip, Av1TransformSize transformSizeContext, int skipContext) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol(skip, this.transformBlockSkip[(int)transformSizeContext][skipContext]); @@ -438,7 +438,7 @@ internal class Av1SymbolEncoder : IDisposable /// Writes the unsigned exponential-Golomb suffix used for coefficient levels beyond the base range. /// /// The nonnegative suffix value. - internal void WriteGolomb(int level) + public void WriteGolomb(int level) { uint x = (uint)level + 1u; int length = (int)Av1Math.Log2_32(x) + 1; @@ -481,7 +481,7 @@ internal class Av1SymbolEncoder : IDisposable /// The active base quantizer index. /// The filter-intra mode when enabled. /// The ordinary intra prediction mode. - internal void WriteTransformType( + public void WriteTransformType( Av1TransformType transformType, Av1TransformSize transformSize, bool useReducedTransformSet, @@ -525,7 +525,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// The segment identifier. /// The context derived from neighboring segment identifiers. - internal void WriteSegmentId(int segmentId, int context) + public void WriteSegmentId(int segmentId, int context) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol(segmentId, this.segmentId[context]); @@ -536,7 +536,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// Indicates whether the block contains no coded transform coefficients. /// The neighboring skip context. - internal void WriteSkip(bool skip, int context) + public void WriteSkip(bool skip, int context) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol(skip, this.skip[context]); @@ -547,7 +547,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// Indicates whether skip mode is selected. /// The neighboring skip-mode context. - internal void WriteSkipMode(bool skip, int context) + public void WriteSkipMode(bool skip, int context) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol(skip, this.skipMode[context]); @@ -558,7 +558,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// The selected filter-intra mode, or the disabled sentinel. /// The block size selecting the enable distribution. - internal void WriteFilterIntraMode(Av1FilterIntraMode filterIntraMode, Av1BlockSize blockSize) + public void WriteFilterIntraMode(Av1FilterIntraMode filterIntraMode, Av1BlockSize blockSize) { ref Av1SymbolWriter w = ref this.writer; bool useFilter = filterIntraMode != Av1FilterIntraMode.AllFilterIntraModes; @@ -573,7 +573,7 @@ internal class Av1SymbolEncoder : IDisposable /// Writes a signed quantizer-index delta value. /// /// The signed quantizer-index delta. - internal void WriteDeltaQuantizerIndex(int deltaQindex) + public void WriteDeltaQuantizerIndex(int deltaQindex) { ref Av1SymbolWriter w = ref this.writer; bool sign = deltaQindex < 0; @@ -603,7 +603,7 @@ internal class Av1SymbolEncoder : IDisposable /// The luma prediction mode. /// The reduced above-mode context. /// The reduced left-mode context. - internal void WriteLumaMode(Av1PredictionMode lumaMode, byte topContext, byte leftContext) + public void WriteLumaMode(Av1PredictionMode lumaMode, byte topContext, byte leftContext) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol((int)lumaMode, this.keyFrameYMode[topContext][leftContext]); @@ -614,7 +614,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// The signed angle delta offset by . /// The directional prediction mode selecting the distribution. - internal void WriteAngleDelta(int angleDelta, Av1PredictionMode context) + public void WriteAngleDelta(int angleDelta, Av1PredictionMode context) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol(angleDelta, this.angleDelta[context - Av1PredictionMode.Vertical]); @@ -625,7 +625,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// The CDEF strength index. /// The number of signaled bits. - internal void WriteCdefStrength(int cdefStrength, int bitCount) + public void WriteCdefStrength(int cdefStrength, int bitCount) { ref Av1SymbolWriter w = ref this.writer; w.WriteLiteral((uint)cdefStrength, bitCount); @@ -637,7 +637,7 @@ internal class Av1SymbolEncoder : IDisposable /// The chroma prediction mode. /// Indicates whether chroma-from-luma is valid for the block. /// The block's luma prediction mode. - internal void WriteChromaMode(Av1PredictionMode chromaMode, bool isChromaFromLumaAllowed, Av1PredictionMode lumaMode) + public void WriteChromaMode(Av1ChromaPredictionMode chromaMode, bool isChromaFromLumaAllowed, Av1PredictionMode lumaMode) { ref Av1SymbolWriter w = ref this.writer; int cflAllowed = isChromaFromLumaAllowed ? 1 : 0; @@ -649,7 +649,7 @@ internal class Av1SymbolEncoder : IDisposable /// /// The packed U/V alpha-magnitude indices. /// The joint U/V sign symbol. - internal void WriteChromaFromLumaAlphas(int chromaFromLumaIndex, int joinedSign) + public void WriteChromaFromLumaAlphas(int chromaFromLumaIndex, int joinedSign) { ref Av1SymbolWriter w = ref this.writer; w.WriteSymbol(joinedSign, this.chromaFromLumaSign); diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolReader.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolReader.cs index f86cbe764..c218f527c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolReader.cs @@ -52,6 +52,11 @@ internal ref struct Av1SymbolReader /// private int count; + /// + /// The adjustment that preserves the logical consumed-bit count after the reader enters implicit zero padding. + /// + private int tellOffset; + /// /// Initializes a new instance of the struct over one entropy-coded span. /// @@ -65,9 +70,40 @@ internal ref struct Av1SymbolReader this.difference = (1U << (DecoderWindowsSize - 1)) - 1; this.range = 0x8000; this.count = -15; + this.tellOffset = 10 - (DecoderWindowsSize - 8); this.Refill(); } + /// + /// Validates that range decoding remained within the bounded tile payload and ended at the required trailing-one bit. + /// + public void ValidateTrailingBits() + { + int consumedBitCount = this.GetConsumedBitCount(); + int consumedByteCount = (consumedBitCount + 7) >> 3; + if (consumedByteCount > this.buffer.Length) + { + throw new InvalidImageContentException("The AV1 tile entropy stream is truncated."); + } + + // The final consumed byte must contain one trailing-one bit at the range decoder's exact stopping position, + // followed only by zero bits. This is the same bounded-stream check performed after libaom decodes a tile. + int trailingOneBit = 128 >> ((consumedBitCount - 1) & 7); + int trailingBitMask = (trailingOneBit << 1) - 1; + if ((this.buffer[consumedByteCount - 1] & trailingBitMask) != trailingOneBit) + { + throw new InvalidImageContentException("The AV1 tile entropy stream has invalid trailing bits."); + } + + for (int i = consumedByteCount; i < this.buffer.Length; i++) + { + if (this.buffer[i] != 0) + { + throw new InvalidImageContentException("The AV1 tile entropy stream has nonzero padding bytes."); + } + } + } + /// /// Reads one symbol and adapts its distribution when CDF updates are enabled. /// @@ -234,7 +270,9 @@ internal ref struct Av1SymbolReader if (position >= end) { // AV1 range decoding permits the final interval to consume implicit zero padding. A large count models - // that padding without advancing beyond the bounded source span or repeatedly attempting to refill it. + // that padding without repeatedly attempting to refill it. Preserve the previous count in tellOffset so + // the logical position continues past the bounded source and truncated payloads remain detectable. + this.tellOffset += LotsOfBits - cnt; cnt = LotsOfBits; } @@ -242,4 +280,11 @@ internal ref struct Av1SymbolReader this.count = cnt; this.position = position; } + + /// + /// Gets the number of entropy bits consumed from the bounded tile payload, including the initial range-coder bit. + /// + /// The logical consumed-bit count. + private readonly int GetConsumedBitCount() + => (this.position * 8) - this.count + this.tellOffset; } diff --git a/src/ImageSharp/Formats/Heif/Av1/IAv1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/IAv1TileReader.cs index 3bbef50bc..7168cdaa1 100644 --- a/src/ImageSharp/Formats/Heif/Av1/IAv1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/IAv1TileReader.cs @@ -4,16 +4,19 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1; /// -/// Interface for reading of image tiles. +/// Defines tile-payload consumption and completion for one coded AV1 frame. /// internal interface IAv1TileReader { /// - /// Read the information for a single tile. + /// Reads one entropy-coded tile payload into the current frame state. /// - /// - /// The bytes of encoded data in the bitstream dedicated to this tile. - /// - /// The index of the tile that is to be read. + /// The bounded bitstream bytes belonging to the tile. + /// The zero-based tile index in raster order. void ReadTile(Span tileData, int tileNum); + + /// + /// Completes the current coded frame after all tile payloads have been read and releases frame-scoped resources. + /// + void CompleteFrame(); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs new file mode 100644 index 000000000..4f86ad80c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs @@ -0,0 +1,234 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Motion; + +/// +/// Stores one AV1 global-motion model in the codec's fixed-point affine matrix domain. +/// +internal struct Av1GlobalMotionParameters +{ + /// + /// The number of fractional bits carried by every stored matrix parameter. + /// + public const int ModelPrecisionBits = 16; + + /// + /// The fixed-point representation of one in the global-motion matrix domain. + /// + public const int ModelScale = 1 << ModelPrecisionBits; + + /// + /// The number of low-order bits removed from the derived shear parameters. + /// + private const int ShearParameterReductionBits = 6; + + /// + /// The number of fractional bits carried by entries in . + /// + private const int ReciprocalPrecisionBits = 14; + + /// + /// The number of divisor-fraction bits used to index . + /// + private const int ReciprocalIndexBits = 8; + + /// + /// The six parameters ordered as horizontal translation, vertical translation, and the four affine coefficients. + /// + private InlineArray6 matrix; + + /// + /// Gets an identity global-motion model. + /// + public static Av1GlobalMotionParameters Identity + { + get + { + Av1GlobalMotionParameters result = default; + result.matrix[2] = ModelScale; + result.matrix[5] = ModelScale; + return result; + } + } + + /// + /// Gets or sets the geometric model represented by the matrix parameters. + /// + public Av1GlobalMotionType Type { get; set; } + + /// + /// Gets the reduced horizontal scale delta used by warped prediction. + /// + public short Alpha { get; private set; } + + /// + /// Gets the reduced horizontal shear used by warped prediction. + /// + public short Beta { get; private set; } + + /// + /// Gets the reduced vertical shear used by warped prediction. + /// + public short Gamma { get; private set; } + + /// + /// Gets the reduced vertical scale delta used by warped prediction. + /// + public short Delta { get; private set; } + + /// + /// Gets a value indicating whether the affine model violates AV1's permitted shear bounds. + /// + public bool IsInvalid { get; private set; } + + /// + /// Gets the fixed-point reciprocal lookup used by AV1's affine shear derivation. + /// + private static ReadOnlySpan ReciprocalTable => + [ + 16384, 16320, 16257, 16194, 16132, 16070, 16009, 15948, 15888, 15828, 15768, + 15709, 15650, 15592, 15534, 15477, 15420, 15364, 15308, 15252, 15197, 15142, + 15087, 15033, 14980, 14926, 14873, 14821, 14769, 14717, 14665, 14614, 14564, + 14513, 14463, 14413, 14364, 14315, 14266, 14218, 14170, 14122, 14075, 14028, + 13981, 13935, 13888, 13843, 13797, 13752, 13707, 13662, 13618, 13574, 13530, + 13487, 13443, 13400, 13358, 13315, 13273, 13231, 13190, 13148, 13107, 13066, + 13026, 12985, 12945, 12906, 12866, 12827, 12788, 12749, 12710, 12672, 12633, + 12596, 12558, 12520, 12483, 12446, 12409, 12373, 12336, 12300, 12264, 12228, + 12193, 12157, 12122, 12087, 12053, 12018, 11984, 11950, 11916, 11882, 11848, + 11815, 11782, 11749, 11716, 11683, 11651, 11619, 11586, 11555, 11523, 11491, + 11460, 11429, 11398, 11367, 11336, 11305, 11275, 11245, 11215, 11185, 11155, + 11125, 11096, 11067, 11038, 11009, 10980, 10951, 10923, 10894, 10866, 10838, + 10810, 10782, 10755, 10727, 10700, 10673, 10645, 10618, 10592, 10565, 10538, + 10512, 10486, 10460, 10434, 10408, 10382, 10356, 10331, 10305, 10280, 10255, + 10230, 10205, 10180, 10156, 10131, 10107, 10082, 10058, 10034, 10010, 9986, + 9963, 9939, 9916, 9892, 9869, 9846, 9823, 9800, 9777, 9754, 9732, 9709, 9687, + 9664, 9642, 9620, 9598, 9576, 9554, 9533, 9511, 9489, 9468, 9447, 9425, 9404, + 9383, 9362, 9341, 9321, 9300, 9279, 9259, 9239, 9218, 9198, 9178, 9158, 9138, + 9118, 9098, 9079, 9059, 9039, 9020, 9001, 8981, 8962, 8943, 8924, 8905, 8886, + 8867, 8849, 8830, 8812, 8793, 8775, 8756, 8738, 8720, 8702, 8684, 8666, 8648, + 8630, 8613, 8595, 8577, 8560, 8542, 8525, 8508, 8490, 8473, 8456, 8439, 8422, + 8405, 8389, 8372, 8355, 8339, 8322, 8306, 8289, 8273, 8257, 8240, 8224, 8208, + 8192, + ]; + + /// + /// Gets or sets a matrix parameter in AV1 affine-transform order. + /// + /// The zero-based matrix parameter index. + /// The fixed-point matrix parameter. + public int this[int index] + { + get => this.matrix[index]; + set => this.matrix[index] = value; + } + + /// + /// Derives the reduced shear parameters and records whether the complete affine model is valid. + /// + public void UpdateShearParameters() + { + Span values = this.matrix; + this.Alpha = 0; + this.Beta = 0; + this.Gamma = 0; + this.Delta = 0; + + if (values[2] <= 0) + { + this.IsInvalid = true; + return; + } + + this.Alpha = (short)Math.Clamp(values[2] - ModelScale, short.MinValue, short.MaxValue); + this.Beta = (short)Math.Clamp(values[3], short.MinValue, short.MaxValue); + + // AV1 derives gamma and delta by multiplying with a fixed-precision reciprocal of the horizontal scale. + // The reciprocal lookup is normative; integer division would produce different warped sample positions. + int reciprocal = ResolveDivisor((uint)values[2], out int reciprocalShift); + long scaledVerticalCoefficient = (long)values[4] * ModelScale * reciprocal; + this.Gamma = (short)Math.Clamp(RoundPowerOf2Signed(scaledVerticalCoefficient, reciprocalShift), short.MinValue, short.MaxValue); + + long scaledCrossCoefficient = (long)values[3] * values[4] * reciprocal; + long verticalScaleDelta = values[5] - RoundPowerOf2Signed(scaledCrossCoefficient, reciprocalShift) - ModelScale; + this.Delta = (short)Math.Clamp(verticalScaleDelta, short.MinValue, short.MaxValue); + + // Warped filtering addresses a coarser parameter grid than the stored affine matrix. Symmetric rounding is + // required here so negative shear values are quantized identically to their positive counterparts. + this.Alpha = ReduceShearParameter(this.Alpha); + this.Beta = ReduceShearParameter(this.Beta); + this.Gamma = ReduceShearParameter(this.Gamma); + this.Delta = ReduceShearParameter(this.Delta); + + // These weighted L1 bounds are the AV1 validity test for the two shear axes. Equality is invalid because the + // warped-filter footprint would no longer remain inside the permitted affine sampling envelope. + this.IsInvalid = + ((4 * Math.Abs((int)this.Alpha)) + (7 * Math.Abs((int)this.Beta)) >= ModelScale) || + ((4 * Math.Abs((int)this.Gamma)) + (4 * Math.Abs((int)this.Delta)) >= ModelScale); + } + + /// + /// Quantizes one signed shear parameter to AV1's warped-filter precision. + /// + /// The full-precision shear parameter. + /// The reduced shear parameter. + private static short ReduceShearParameter(short value) + => (short)(RoundPowerOf2Signed(value, ShearParameterReductionBits) * (1 << ShearParameterReductionBits)); + + /// + /// Resolves a positive divisor into AV1's fixed-point reciprocal representation. + /// + /// The positive divisor. + /// Receives the reciprocal's binary scale. + /// The fixed-point reciprocal multiplier. + private static int ResolveDivisor(uint divisor, out int shift) + { + // Normalize the divisor around its highest set bit, then quantize the remaining fraction to the normative + // eight-bit table index. Adding the table's fourteen fractional bits yields the scale used by the caller's + // rounded multiply instead of a platform-dependent integer division. + shift = BitOperations.Log2(divisor); + int remainder = (int)(divisor - (1U << shift)); + int reciprocalIndex = shift > ReciprocalIndexBits + ? RoundPowerOf2(remainder, shift - ReciprocalIndexBits) + : remainder << (ReciprocalIndexBits - shift); + + shift += ReciprocalPrecisionBits; + return ReciprocalTable[reciprocalIndex]; + } + + /// + /// Divides a nonnegative integer by a power of two with nearest-integer rounding. + /// + /// The nonnegative value. + /// The base-two divisor exponent. + /// The rounded quotient. + private static int RoundPowerOf2(int value, int bitCount) + => (value + ((1 << bitCount) >> 1)) >> bitCount; + + /// + /// Divides a signed integer by a power of two with symmetric nearest-integer rounding. + /// + /// The signed value. + /// The base-two divisor exponent. + /// The rounded quotient. + private static long RoundPowerOf2Signed(long value, int bitCount) + => value < 0 + ? -(((-value) + ((1L << bitCount) >> 1)) >> bitCount) + : (value + ((1L << bitCount) >> 1)) >> bitCount; + + /// + /// Provides inline storage for the six parameters in an AV1 affine matrix. + /// + /// The stored parameter type. + [InlineArray(6)] + private struct InlineArray6 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionType.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionType.cs new file mode 100644 index 000000000..59cb14d7f --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionType.cs @@ -0,0 +1,30 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Motion; + +/// +/// Identifies the geometric model carried by AV1 global-motion parameters. +/// +internal enum Av1GlobalMotionType : byte +{ + /// + /// No geometric displacement is applied. + /// + Identity = 0, + + /// + /// Horizontal and vertical translation are applied. + /// + Translation = 1, + + /// + /// Translation, rotation, and uniform zoom are applied. + /// + RotationZoom = 2, + + /// + /// A general six-parameter affine transformation is applied. + /// + Affine = 3 +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1IntraBlockCopy.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1IntraBlockCopy.cs index d53b8b4b2..06aea4e91 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1IntraBlockCopy.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1IntraBlockCopy.cs @@ -36,7 +36,7 @@ internal static class Av1IntraBlockCopy /// Reusable storage for the corresponding spatial weights. /// The nearest nonzero spatial candidate, or the normative tile-relative fallback. public static Av1MotionVector FindReference( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo, int superblockModeInfoSize, Span candidates, @@ -69,17 +69,17 @@ internal static class Av1IntraBlockCopy int processedColumns = 0; if (Math.Abs(maximumRowOffset) >= 1) { - ScanRow(partitionInfo, -1, maximumRowOffset, candidates, weights, ref candidateCount, ref processedRows); + ScanRow(ref partitionInfo, -1, maximumRowOffset, candidates, weights, ref candidateCount, ref processedRows); } if (Math.Abs(maximumColumnOffset) >= 1) { - ScanColumn(partitionInfo, -1, maximumColumnOffset, candidates, weights, ref candidateCount, ref processedColumns); + ScanColumn(ref partitionInfo, -1, maximumColumnOffset, candidates, weights, ref candidateCount, ref processedColumns); } - if (HasTopRight(partitionInfo, superblockModeInfoSize)) + if (HasTopRight(ref partitionInfo, superblockModeInfoSize)) { - AddBlock(partitionInfo, -1, width, tileInfo, candidates, weights, ref candidateCount); + AddBlock(ref partitionInfo, -1, width, tileInfo, candidates, weights, ref candidateCount); } int nearestCandidateCount = candidateCount; @@ -90,19 +90,19 @@ internal static class Av1IntraBlockCopy // The top-left sample begins the outer search region. Sorting the adjacent and outer regions independently // preserves libaom's nearest/near ordering while still accumulating repeated vectors across both regions. - AddBlock(partitionInfo, -1, -1, tileInfo, candidates, weights, ref candidateCount); + AddBlock(ref partitionInfo, -1, -1, tileInfo, candidates, weights, ref candidateCount); for (int index = 2; index <= ReferenceSearchDistance; index++) { int rowOffset = -(index << 1) + 1 + rowAdjustment; int columnOffset = -(index << 1) + 1 + columnAdjustment; if (Math.Abs(rowOffset) <= Math.Abs(maximumRowOffset) && Math.Abs(rowOffset) > processedRows) { - ScanRow(partitionInfo, rowOffset, maximumRowOffset, candidates, weights, ref candidateCount, ref processedRows); + ScanRow(ref partitionInfo, rowOffset, maximumRowOffset, candidates, weights, ref candidateCount, ref processedRows); } if (Math.Abs(columnOffset) <= Math.Abs(maximumColumnOffset) && Math.Abs(columnOffset) > processedColumns) { - ScanColumn(partitionInfo, columnOffset, maximumColumnOffset, candidates, weights, ref candidateCount, ref processedColumns); + ScanColumn(ref partitionInfo, columnOffset, maximumColumnOffset, candidates, weights, ref candidateCount, ref processedColumns); } } @@ -138,7 +138,7 @@ internal static class Av1IntraBlockCopy /// The active tile boundaries. /// The sequence-level superblock and chroma configuration. /// when the complete source block is a permitted reference; otherwise, . - public static bool IsValid(Av1MotionVector vector, Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo, ObuSequenceHeader sequenceHeader) + public static bool IsValid(Av1MotionVector vector, ref Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo, ObuSequenceHeader sequenceHeader) { const int eighthSampleScale = 8; const int modeInfoSampleSize = 1 << Av1Constants.ModeInfoSizeLog2; @@ -207,7 +207,7 @@ internal static class Av1IntraBlockCopy /// Scans a mode-information row using AV1's block-size-dependent steps and weights. /// private static void ScanRow( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, int rowOffset, int maximumRowOffset, Span candidates, @@ -261,7 +261,7 @@ internal static class Av1IntraBlockCopy /// Scans a mode-information column using AV1's block-size-dependent steps and weights. /// private static void ScanColumn( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, int columnOffset, int maximumColumnOffset, Span candidates, @@ -315,7 +315,7 @@ internal static class Av1IntraBlockCopy /// Adds the intra-block-copy vector at one tile-relative search position. /// private static void AddBlock( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, int rowOffset, int columnOffset, Av1TileInfo tileInfo, @@ -400,7 +400,7 @@ internal static class Av1IntraBlockCopy /// /// Determines whether the current partition is parsed after the block at its top-right search position. /// - private static bool HasTopRight(Av1PartitionInfo partitionInfo, int superblockModeInfoSize) + private static bool HasTopRight(ref Av1PartitionInfo partitionInfo, int superblockModeInfoSize) { int width = partitionInfo.ModeInfo.BlockSize.Get4x4WideCount(); int height = partitionInfo.ModeInfo.BlockSize.Get4x4HighCount(); diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs index 0fcddb47c..b76d94785 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFilmGrainParameters.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; /// @@ -9,167 +11,282 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuFilmGrainParameters { /// - /// Gets or sets a value indicating whether film grain should be added to this frame. A value equal to false specifies that film - /// grain should not be added. + /// Stores the luma scaling-point coordinates without a per-frame array allocation. + /// + private InlineArray14 pointYValue; + + /// + /// Stores the luma scaling-point values without a per-frame array allocation. + /// + private InlineArray14 pointYScaling; + + /// + /// Stores the blue-difference scaling-point coordinates without a per-frame array allocation. + /// + private InlineArray10 pointCbValue; + + /// + /// Stores the blue-difference scaling-point values without a per-frame array allocation. + /// + private InlineArray10 pointCbScaling; + + /// + /// Stores the red-difference scaling-point coordinates without a per-frame array allocation. + /// + private InlineArray10 pointCrValue; + + /// + /// Stores the red-difference scaling-point values without a per-frame array allocation. + /// + private InlineArray10 pointCrScaling; + + /// + /// Stores the luma autoregressive coefficients without a per-frame array allocation. + /// + private InlineArray24 arCoeffsYPlus128; + + /// + /// Stores the blue-difference autoregressive coefficients without a per-frame array allocation. + /// + private InlineArray25 arCoeffsCbPlus128; + + /// + /// Stores the red-difference autoregressive coefficients without a per-frame array allocation. + /// + private InlineArray25 arCoeffsCrPlus128; + + /// + /// Gets or sets a value indicating whether film grain is applied to the displayed frame. /// public bool ApplyGrain { get; set; } /// - /// Gets or sets GrainSeed. This value specifies the starting value for the pseudo-random numbers used during film grain synthesis. + /// Gets or sets the 16-bit seed that initializes pseudo-random film-grain synthesis for this frame. /// public uint GrainSeed { get; set; } /// - /// Gets or sets a value indicating whether a new set of parameters should be sent. A value equal to false means that the - /// previous set of parameters should be used. + /// Gets or sets a value indicating whether this frame signals a complete parameter set instead of inheriting one. /// public bool UpdateGrain { get; set; } /// - /// Gets or sets FilmGrainParamsRefIdx. Indicates which reference frame contains the film grain parameters to be used for this frame. - /// It is a requirement of bitstream conformance that FilmGrainParamsRefIdx is equal to ref_frame_idx[ j ] for some value - /// of j in the range 0 to REFS_PER_FRAME - 1. + /// Gets or sets the physical reference-map index from which this frame inherited its film-grain parameters. + /// The index must match one of the frame's seven selected inter-reference slots. /// - public uint FilmGrainParamsRefidx { get; set; } + public uint FilmGrainParamsRefIdx { get; set; } /// - /// Gets or sets NumYPoints. Specifies the number of points for the piece-wise linear scaling function of the luma component. - /// It is a requirement of bitstream conformance that NumYPoints is less than or equal to 14. + /// Gets or sets the number of active luma scaling points in the inclusive range zero through fourteen. /// public uint NumYPoints { get; set; } /// - /// Gets or sets PointYValue. Represents the x (luma value) coordinate for the i-th point of the piecewise linear scaling function for - /// luma component.The values are signaled on the scale of 0..255. (In case of 10 bit video, these values correspond to - /// luma values divided by 4. In case of 12 bit video, these values correspond to luma values divided by 16.) - /// - /// If i is greater than 0, it is a requirement of bitstream conformance that point_y_value[ i ] is greater than point_y_value[ i - 1] (this ensures the x coordinates are specified in increasing order). + /// Gets the fourteen-entry storage for the luma scaling-point coordinates. /// - public uint[]? PointYValue { get; set; } + /// + /// Only the first entries are active. Coordinates use the eight-bit scale and must be + /// strictly increasing; 10-bit and 12-bit sample values are divided by four and sixteen respectively. + /// + public Span PointYValue => this.pointYValue; /// - /// Gets or sets PointYScaling. Represents the scaling (output) value for the i-th point of the piecewise linear scaling function for luma component. + /// Gets the fourteen-entry storage for the luma scaling-point output values. /// - public uint[]? PointYScaling { get; set; } + /// Only the first entries are active. + public Span PointYScaling => this.pointYScaling; /// - /// Gets or sets a value indicating whether the chroma scaling is inferred from the luma scaling. + /// Gets or sets a value indicating whether both chroma scaling functions are derived from luma samples. /// public bool ChromaScalingFromLuma { get; set; } /// - /// Gets or sets NumCbPoints. Specifies the number of points for the piece-wise linear scaling function of the cb component. - /// It is a requirement of bitstream conformance that NumCbPoints is less than or equal to 10. + /// Gets or sets the number of active blue-difference scaling points in the inclusive range zero through ten. /// public uint NumCbPoints { get; set; } /// - /// Gets or sets NumCrPoints. Specifies represents the number of points for the piece-wise linear scaling function of the cr component. - /// It is a requirement of bitstream conformance that NumCrPoints is less than or equal to 10. + /// Gets or sets the number of active red-difference scaling points in the inclusive range zero through ten. /// public uint NumCrPoints { get; set; } /// - /// Gets or sets PointCbValue. Represents the x coordinate for the i-th point of the piece-wise linear scaling function for cb - /// component.The values are signaled on the scale of 0..255. - /// If i is greater than 0, it is a requirement of bitstream conformance that point_cb_value[ i ] is greater than point_cb_value[ i - 1 ]. + /// Gets the ten-entry storage for the blue-difference scaling-point coordinates. /// - public uint[]? PointCbValue { get; set; } + /// Only the first entries are active, and active coordinates must be strictly increasing. + public Span PointCbValue => this.pointCbValue; /// - /// Gets or sets PointCbScaling. Represents the scaling (output) value for the i-th point of the piecewise linear scaling function for cb component. + /// Gets the ten-entry storage for the blue-difference scaling-point output values. /// - public uint[]? PointCbScaling { get; set; } + /// Only the first entries are active. + public Span PointCbScaling => this.pointCbScaling; /// - /// Gets or sets PointCrValue. Represents the x coordinate for the i-th point of the piece-wise linear scaling function for cr component. - /// The values are signaled on the scale of 0..255. - /// If i is greater than 0, it is a requirement of bitstream conformance that point_cr_value[ i ] is greater than point_cr_value[ i - 1 ]. + /// Gets the ten-entry storage for the red-difference scaling-point coordinates. /// - public uint[]? PointCrValue { get; set; } + /// Only the first entries are active, and active coordinates must be strictly increasing. + public Span PointCrValue => this.pointCrValue; /// - /// Gets or sets PointCrScaling. Represents the scaling (output) value for the i-th point of the piecewise linear scaling function for cr component. + /// Gets the ten-entry storage for the red-difference scaling-point output values. /// - public uint[]? PointCrScaling { get; set; } + /// Only the first entries are active. + public Span PointCrScaling => this.pointCrScaling; /// - /// Gets or sets GrainScalingMinus8. represents the shift – 8 applied to the values of the chroma component. The - /// grain_scaling_minus_8 can take values of 0..3 and determines the range and quantization step of the standard deviation of film grain. + /// Gets or sets the scaling-function shift minus eight. Values from zero through three select an effective shift + /// from eight through eleven for every luma and chroma scaling value. /// public uint GrainScalingMinus8 { get; set; } /// - /// Gets or sets ArCoeffLag. Specifies the number of auto-regressive coefficients for luma and chroma. + /// Gets or sets the autoregressive neighborhood lag in the inclusive range zero through three. /// public uint ArCoeffLag { get; set; } /// - /// Gets or sets ArCoeffsYPlus128. Specifies auto-regressive coefficients used for the Y plane. + /// Gets the twenty-four-entry storage for biased luma autoregressive coefficients. /// - public uint[]? ArCoeffsYPlus128 { get; set; } + /// The active entry count is 2 * ArCoeffLag * (ArCoeffLag + 1). + public Span ArCoeffsYPlus128 => this.arCoeffsYPlus128; /// - /// Gets or sets ArCoeffsCbPlus128. Specifies auto-regressive coefficients used for the U plane. + /// Gets the twenty-five-entry storage for biased blue-difference autoregressive coefficients. /// - public uint[]? ArCoeffsCbPlus128 { get; set; } + /// The active entry count includes one additional luma coefficient when luma scaling points are present. + public Span ArCoeffsCbPlus128 => this.arCoeffsCbPlus128; /// - /// Gets or sets ArCoeffsCrPlus128. Specifies auto-regressive coefficients used for the V plane. + /// Gets the twenty-five-entry storage for biased red-difference autoregressive coefficients. /// - public uint[]? ArCoeffsCrPlus128 { get; set; } + /// The active entry count includes one additional luma coefficient when luma scaling points are present. + public Span ArCoeffsCrPlus128 => this.arCoeffsCrPlus128; /// - /// Gets or sets ArCoeffShiftMinus6. Specifies the range of the auto-regressive coefficients. Values of 0, 1, 2, and 3 correspond to the - /// ranges for auto-regressive coefficients of[-2, 2), [-1, 1), [-0.5, 0.5) and [-0.25, 0.25) respectively. + /// Gets or sets the autoregressive coefficient shift minus six in the inclusive range zero through three. /// public uint ArCoeffShiftMinus6 { get; set; } /// - /// Gets or sets GrainScaleShift. Specifies how much the Gaussian random numbers should be scaled down during the grain synthesis process. + /// Gets or sets the right shift applied to generated Gaussian grain samples in the inclusive range zero through three. /// public uint GrainScaleShift { get; set; } /// - /// Gets or sets CbMult. Represents a multiplier for the cb component used in derivation of the input index to the cb component scaling function. + /// Gets or sets the 8-bit blue-difference sample multiplier used to derive the chroma scaling index. /// public uint CbMult { get; set; } /// - /// Gets or sets CbLumaMult. Represents a multiplier for the average luma component used in derivation of the input index to the cb component scaling function. + /// Gets or sets the 8-bit average-luma multiplier used to derive the blue-difference scaling index. /// public uint CbLumaMult { get; set; } /// - /// Gets or sets CbOffset. Represents an offset used in derivation of the input index to the cb component scaling function. + /// Gets or sets the 9-bit offset used to derive the blue-difference scaling index. /// public uint CbOffset { get; set; } /// - /// Gets or sets CrMult. Represents a multiplier for the cr component used in derivation of the input index to the cr component scaling function. + /// Gets or sets the 8-bit red-difference sample multiplier used to derive the chroma scaling index. /// public uint CrMult { get; set; } /// - /// Gets or sets CrLumaMult. Represents a multiplier for the average luma component used in derivation of the input index to the cr component scaling function. + /// Gets or sets the 8-bit average-luma multiplier used to derive the red-difference scaling index. /// public uint CrLumaMult { get; set; } /// - /// Gets or sets CrOffset. Represents an offset used in derivation of the input index to the cr component scaling function. + /// Gets or sets the 9-bit offset used to derive the red-difference scaling index. /// public uint CrOffset { get; set; } /// - /// Gets or sets a value indicating whether the overlap between film grain blocks shall be applied. OverlapFlag equal to false - /// indicates that the overlap between film grain blocks shall not be applied. + /// Gets or sets a value indicating whether neighboring film-grain blocks are blended across their boundaries. /// public bool OverlapFlag { get; set; } /// - /// Gets or sets a value indicating whether clipping to the restricted (studio) range shall be applied to the sample - /// values after adding the film grain(see the semantics for color_range for an explanation of studio swing). - /// ClipToRestrictedRange equal to false indicates that clipping to the full range shall be applied to the sample values after adding the film grain. + /// Gets or sets a value indicating whether grained samples are clipped to the restricted range instead of the full range. /// public bool ClipToRestrictedRange { get; set; } + + /// + /// Replaces the complete film-grain parameter set with values retained by a reference frame. + /// + /// The retained reference-frame parameters. + public void CopyFrom(ObuFilmGrainParameters source) + { + this.ApplyGrain = source.ApplyGrain; + this.GrainSeed = source.GrainSeed; + this.UpdateGrain = source.UpdateGrain; + this.FilmGrainParamsRefIdx = source.FilmGrainParamsRefIdx; + this.NumYPoints = source.NumYPoints; + this.pointYValue = source.pointYValue; + this.pointYScaling = source.pointYScaling; + this.ChromaScalingFromLuma = source.ChromaScalingFromLuma; + this.NumCbPoints = source.NumCbPoints; + this.NumCrPoints = source.NumCrPoints; + this.pointCbValue = source.pointCbValue; + this.pointCbScaling = source.pointCbScaling; + this.pointCrValue = source.pointCrValue; + this.pointCrScaling = source.pointCrScaling; + this.GrainScalingMinus8 = source.GrainScalingMinus8; + this.ArCoeffLag = source.ArCoeffLag; + this.arCoeffsYPlus128 = source.arCoeffsYPlus128; + this.arCoeffsCbPlus128 = source.arCoeffsCbPlus128; + this.arCoeffsCrPlus128 = source.arCoeffsCrPlus128; + this.ArCoeffShiftMinus6 = source.ArCoeffShiftMinus6; + this.GrainScaleShift = source.GrainScaleShift; + this.CbMult = source.CbMult; + this.CbLumaMult = source.CbLumaMult; + this.CbOffset = source.CbOffset; + this.CrMult = source.CrMult; + this.CrLumaMult = source.CrLumaMult; + this.CrOffset = source.CrOffset; + this.OverlapFlag = source.OverlapFlag; + this.ClipToRestrictedRange = source.ClipToRestrictedRange; + } + + /// + /// Provides inline storage for the maximum luma autoregressive coefficient count. + /// + /// The stored value type. + [InlineArray(24)] + private struct InlineArray24 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } + + /// + /// Provides inline storage for the ten scaling points permitted on either chroma plane. + /// + /// The stored value type. + [InlineArray(10)] + private struct InlineArray10 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } + + /// + /// Provides inline storage for the maximum autoregressive coefficient count of either chroma plane. + /// + /// The stored value type. + [InlineArray(25)] + private struct InlineArray25 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs index 73084c3df..86c4f42ac 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -1,6 +1,9 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; @@ -21,10 +24,30 @@ internal class ObuFrameHeader private InlineArray8 referenceOrderHint; /// - /// Stores the reference-map index selected for each of the eight inter references. + /// Stores the frame identifier associated with each of the eight reference-map slots. + /// + private InlineArray8 referenceFrameId; + + /// + /// Stores the reference-map slot selected for each of the seven inter reference types. /// private InlineArray8 referenceFrameIndex; + /// + /// Stores the global-motion model associated with each of the seven inter reference types. + /// + private InlineArray7 globalMotionParameters; + + /// + /// Gets or sets the temporal-layer identifier carried by the primary frame-header OBU. + /// + public int TemporalId { get; set; } + + /// + /// Gets or sets the spatial-layer identifier carried by the primary frame-header OBU. + /// + public int SpatialId { get; set; } + /// /// Gets or sets a value indicating whether motion vectors use integer-sample precision. /// @@ -45,6 +68,16 @@ internal class ObuFrameHeader /// public bool AllowHighPrecisionMotionVector { get; set; } + /// + /// Gets or sets the frame-level interpolation filter used for inter prediction. + /// + public Av1InterpolationFilter InterpolationFilter { get; set; } + + /// + /// Gets or sets a value indicating whether inter blocks may select a non-translational motion mode. + /// + public bool IsMotionModeSwitchable { get; set; } + /// /// Gets or sets the decoded tile layout. /// @@ -148,103 +181,172 @@ internal class ObuFrameHeader /// /// Gets or sets the decoded and rendered frame dimensions. /// - internal ObuFrameSize FrameSize { get; set; } = new ObuFrameSize(); + public ObuFrameSize FrameSize { get; set; } = new ObuFrameSize(); /// /// Gets or sets the frame width in mode-information units. /// - internal int ModeInfoColumnCount { get; set; } + public int ModeInfoColumnCount { get; set; } /// /// Gets or sets the frame height in mode-information units. /// - internal int ModeInfoRowCount { get; set; } + public int ModeInfoRowCount { get; set; } /// /// Gets or sets a value indicating whether an existing reference frame is displayed without decoding a new frame. /// - internal bool ShowExistingFrame { get; set; } + public bool ShowExistingFrame { get; set; } /// /// Gets or sets the coded frame type. /// - internal ObuFrameType FrameType { get; set; } - - /// - /// Gets the validity state of each reference-frame slot. - /// - /// The mutable eight-entry reference-validity table. - public Span GetReferenceValidity() => this.referenceValid; - - /// - /// Gets the multi-bit order hint associated with each reference-frame slot. - /// - /// The mutable eight-entry reference-order-hint table. - public Span GetReferenceOrderHints() => this.referenceOrderHint; + public ObuFrameType FrameType { get; set; } /// /// Gets or sets a value indicating whether the decoded frame is immediately displayed. /// - internal bool ShowFrame { get; set; } + public bool ShowFrame { get; set; } /// /// Gets or sets a value indicating whether the frame may be displayed by a later header. /// - internal bool ShowableFrame { get; set; } + public bool ShowableFrame { get; set; } /// /// Gets or sets the reference map index selected when showing an existing frame. /// - internal uint FrameToShowMapIdx { get; set; } + public uint FrameToShowMapIdx { get; set; } /// /// Gets or sets the display frame identifier. /// - internal uint DisplayFrameId { get; set; } + public uint DisplayFrameId { get; set; } /// /// Gets or sets a value indicating whether the frame can be decoded without state from earlier frames. /// - internal bool ErrorResilientMode { get; set; } + public bool ErrorResilientMode { get; set; } /// /// Gets or sets a value indicating whether screen-content coding tools are permitted. /// - internal bool AllowScreenContentTools { get; set; } + public bool AllowScreenContentTools { get; set; } /// /// Gets or sets a value indicating whether probability adaptation is disabled for this frame. /// - internal bool DisableCdfUpdate { get; set; } + public bool DisableCdfUpdate { get; set; } /// /// Gets or sets the identifier of the current frame. /// - internal uint CurrentFrameId { get; set; } + public uint CurrentFrameId { get; set; } /// - /// Gets the reference-map index selected for each inter reference. + /// Gets or sets the frame order hint. /// - /// The mutable eight-entry reference-frame-index table. - public Span GetReferenceFrameIndices() => this.referenceFrameIndex; + public uint OrderHint { get; set; } /// - /// Gets or sets the frame order hint. + /// Gets or sets the zero-based inter-reference type that supplies the initial frame context, or the no-reference + /// sentinel. /// - internal uint OrderHint { get; set; } + public uint PrimaryReferenceFrame { get; set; } = Av1Constants.PrimaryReferenceFrameNone; /// - /// Gets or sets the reference frame supplying the initial probability context. + /// Gets or sets the resolved reference-map slot supplying the initial frame context, or when + /// the frame uses the default context. /// - internal uint PrimaryReferenceFrame { get; set; } = Av1Constants.PrimaryReferenceFrameNone; + public byte? PrimaryReferenceSlot { get; set; } /// /// Gets or sets the bit mask of reference slots refreshed by this frame. /// - internal uint RefreshFrameFlags { get; set; } + public uint RefreshFrameFlags { get; set; } /// /// Gets or sets the presentation time signaled by temporal point information. /// - internal uint FramePresentationTime { get; set; } + public uint FramePresentationTime { get; set; } + + /// + /// Gets the validity state of each reference-frame slot. + /// + /// The mutable eight-entry reference-validity table. + public Span GetReferenceValidity() => this.referenceValid; + + /// + /// Gets the multi-bit order hint associated with each reference-frame slot. + /// + /// The mutable eight-entry reference-order-hint table. + public Span GetReferenceOrderHints() => this.referenceOrderHint; + + /// + /// Gets the frame identifier associated with each reference-map slot. + /// + /// The mutable eight-entry reference-frame-identifier table. + public Span GetReferenceFrameIds() => this.referenceFrameId; + + /// + /// Gets the reference-map slot selected for each inter reference type. + /// + /// The mutable seven-entry inter-reference-map table. + public Span GetReferenceFrameIndices() => this.referenceFrameIndex[..Av1Constants.ReferencesPerFrame]; + + /// + /// Gets the global-motion model associated with each canonical inter reference type from LAST through ALTREF. + /// + /// The mutable seven-entry global-motion parameter table. + public Span GetGlobalMotionParameters() => this.globalMotionParameters; + + /// + /// Invalidates retained reference slots whose frame identifiers fall outside the permitted backwards window. + /// + /// The number of bits in the modulo frame-identifier domain. + /// The number of bits used to signal reference-frame identifier deltas. + public void MarkReferenceFrames(int frameIdLength, int deltaFrameIdLength) + { + uint referenceWindow = 1U << deltaFrameIdLength; + uint frameIdModulus = 1U << frameIdLength; + Span referenceFrameIds = this.GetReferenceFrameIds(); + Span referenceValidity = this.GetReferenceValidity(); + + for (int slot = 0; slot < Av1Constants.ReferenceFrameCount; slot++) + { + uint referenceFrameId = referenceFrameIds[slot]; + if (this.CurrentFrameId > referenceWindow) + { + // Without wraparound, a retained identifier is valid only in the closed interval ending at the + // current identifier and extending referenceWindow values backwards. + if (referenceFrameId > this.CurrentFrameId || referenceFrameId < this.CurrentFrameId - referenceWindow) + { + referenceValidity[slot] = false; + } + } + else + { + // When the backwards window crosses zero, valid identifiers occupy both ends of the modulo domain. + // Only the open interval between the current identifier and the wrapped lower bound is invalid. + uint wrappedLowerBound = frameIdModulus + this.CurrentFrameId - referenceWindow; + if (referenceFrameId > this.CurrentFrameId && referenceFrameId < wrappedLowerBound) + { + referenceValidity[slot] = false; + } + } + } + } + + /// + /// Provides inline storage for the seven canonical AV1 inter reference types. + /// + /// The stored parameter type. + [InlineArray(Av1Constants.ReferencesPerFrame)] + private struct InlineArray7 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameReferenceState.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameReferenceState.cs new file mode 100644 index 000000000..f8f3533f7 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameReferenceState.cs @@ -0,0 +1,101 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; + +/// +/// Stores the uncompressed-header reference state retained by one AV1 OBU reader session. +/// +/// +/// This state describes the eight reference-map slots but does not own reconstructed sample buffers. Pixel ownership +/// remains with the decoder's reference-frame store and is committed before this syntax state is completed. CDF, +/// segmentation, loop-filter, motion, and layer metadata remain on that retained frame owner; the current header's +/// resolved primary-reference slot selects the shared owner instead of duplicating those values here. +/// +internal struct ObuFrameReferenceState +{ + /// + /// Stores whether each of the eight reference-map slots can be selected by a later frame. + /// + private InlineArray8 referenceValidity; + + /// + /// Stores the frame identifier associated with each of the eight reference-map slots. + /// + private InlineArray8 referenceFrameIds; + + /// + /// Stores the order hint associated with each of the eight reference-map slots. + /// + private InlineArray8 referenceOrderHints; + + /// + /// Gets a value indicating whether a completed frame identifier is available for the next header. + /// + public bool HasCurrentFrameId { get; private set; } + + /// + /// Gets the frame identifier of the most recently completed frame. + /// + public uint CurrentFrameId { get; private set; } + + /// + /// Copies the completed reference-map state into a newly created frame header. + /// + /// The frame header that will parse and derive state from the retained map. + public void InitializeFrameHeader(ObuFrameHeader frameHeader) + { + ReadOnlySpan referenceValidity = this.referenceValidity; + ReadOnlySpan referenceFrameIds = this.referenceFrameIds; + ReadOnlySpan referenceOrderHints = this.referenceOrderHints; + + // Only the eight retained-slot tables cross a frame boundary. The seven inter-reference roles are signaled or + // derived afresh for each frame, and the primary context source is resolved from that per-frame mapping. + referenceValidity.CopyTo(frameHeader.GetReferenceValidity()); + referenceFrameIds.CopyTo(frameHeader.GetReferenceFrameIds()); + referenceOrderHints.CopyTo(frameHeader.GetReferenceOrderHints()); + } + + /// + /// Publishes the reference-map transition produced by a successfully completed frame. + /// + /// The completed frame header whose refresh mask selects the replaced slots. + /// + /// A value indicating whether the sequence carries modulo frame identifiers. + /// + public void CompleteFrame(ObuFrameHeader frameHeader, bool frameIdNumbersPresent) + { + Span referenceValidity = frameHeader.GetReferenceValidity(); + Span referenceFrameIds = frameHeader.GetReferenceFrameIds(); + Span referenceOrderHints = frameHeader.GetReferenceOrderHints(); + + // Refresh is published only at this successful completion boundary. Updating the completed header first keeps + // the same object retained by the reconstructed frame owner synchronized with the next parser-session snapshot. + for (int slot = 0; slot < Av1Constants.ReferenceFrameCount; slot++) + { + if ((frameHeader.RefreshFrameFlags & (1U << slot)) != 0) + { + referenceValidity[slot] = true; + referenceFrameIds[slot] = frameHeader.CurrentFrameId; + referenceOrderHints[slot] = frameHeader.OrderHint; + } + } + + referenceValidity.CopyTo(this.referenceValidity); + referenceFrameIds.CopyTo(this.referenceFrameIds); + referenceOrderHints.CopyTo(this.referenceOrderHints); + + if (frameIdNumbersPresent) + { + // libaom keeps one current_frame_id in decoder-session state. The following header snapshots this value as + // its previous identifier before consuming its own current_frame_id syntax. + this.CurrentFrameId = frameHeader.CurrentFrameId; + this.HasCurrentFrameId = true; + } + } + + /// + /// Clears the completed frame identifier and every retained reference-map slot. + /// + public void Reset() => this = default; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs index 3f5ef3e4b..63c713e48 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameSize.cs @@ -11,30 +11,30 @@ internal class ObuFrameSize /// /// Gets or sets the coded frame width. /// - internal int FrameWidth { get; set; } + public int FrameWidth { get; set; } /// /// Gets or sets the coded frame height. /// - internal int FrameHeight { get; set; } + public int FrameHeight { get; set; } /// /// Gets or sets the denominator used by AV1 super-resolution scaling. /// - internal int SuperResolutionDenominator { get; set; } + public int SuperResolutionDenominator { get; set; } /// /// Gets or sets the frame width after super-resolution upscaling. /// - internal int SuperResolutionUpscaledWidth { get; set; } + public int SuperResolutionUpscaledWidth { get; set; } /// /// Gets or sets the intended display width. /// - internal int RenderWidth { get; set; } + public int RenderWidth { get; set; } /// /// Gets or sets the intended display height. /// - internal int RenderHeight { get; set; } + public int RenderHeight { get; set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs index 523291960..06f93f00e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOrderHintInfo.cs @@ -11,20 +11,20 @@ internal class ObuOrderHintInfo /// /// Gets or sets a value indicating whether order hints are enabled. /// - public bool EnableOrderHint { get; internal set; } + public bool EnableOrderHint { get; set; } /// /// Gets or sets a value indicating whether joint compound prediction is enabled. /// - internal bool EnableJointCompound { get; set; } + public bool EnableJointCompound { get; set; } /// /// Gets or sets a value indicating whether reference-frame motion vectors are enabled. /// - internal bool EnableReferenceFrameMotionVectors { get; set; } + public bool EnableReferenceFrameMotionVectors { get; set; } /// /// Gets or sets the number of bits used to encode order hints. /// - internal int OrderHintBits { get; set; } + public int OrderHintBits { get; set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index 9036cfb1c..e00ccf668 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -1,7 +1,10 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; +using SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; @@ -11,11 +14,72 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; /// internal class ObuReader { + /// + /// The number of bits used to address one of AV1's eight reference-map slots. + /// + private const int ReferenceFrameIndexBits = 3; + + /// + /// The initial finite-subexponential group width used by every global-motion parameter. + /// + private const int GlobalMotionSubexponentialGroupBitCount = 3; + + /// + /// The finite signed-domain size parameter for coded global-motion affine coefficients. + /// + private const int GlobalMotionAlphaValueMagnitude = (1 << 12) + 1; + + /// + /// The number of fractional bits carried by coded global-motion affine coefficients. + /// + private const int GlobalMotionAlphaPrecisionBits = 15; + + /// + /// The precision increase from a coded affine coefficient to the stored global-motion matrix. + /// + private const int GlobalMotionAlphaPrecisionDifference = + Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionAlphaPrecisionBits; + + /// + /// The scale factor that restores a coded affine coefficient to the global-motion matrix precision. + /// + private const int GlobalMotionAlphaDecodeFactor = 1 << GlobalMotionAlphaPrecisionDifference; + + /// + /// The signed magnitude bit count of a general affine model's translation components. + /// + private const int GlobalMotionAbsoluteTranslationBits = 12; + + /// + /// The signed magnitude bit count of a translation-only model before precision adjustment. + /// + private const int GlobalMotionAbsoluteTranslationOnlyBits = 9; + + /// + /// The number of fractional bits carried by general affine translation components. + /// + private const int GlobalMotionTranslationPrecisionBits = 6; + + /// + /// The number of fractional bits carried by translation-only components. + /// + private const int GlobalMotionTranslationOnlyPrecisionBits = 3; + /// /// The zero-based sequence-header operating-point index selected by the container. /// private readonly byte operatingPointIndex; + /// + /// The reconstructed frames retained by the owning decoder for inter-frame syntax and prediction. + /// + private readonly Av1ReferenceFrameStore? referenceFrames; + + /// + /// The completed frame-identifier, validity, and order-hint state retained across frame headers in this session. + /// + private ObuFrameReferenceState frameReferenceState; + /// /// The tile reader created for the current coded frame. /// @@ -27,7 +91,8 @@ internal class ObuReader private uint currentOperatingPointIdc; /// - /// Initializes a new instance of the class using operating-point index zero. + /// Initializes a new instance of the class using operating-point index zero without a + /// reconstructed reference map. /// public ObuReader() : this(0) @@ -35,12 +100,25 @@ internal class ObuReader } /// - /// Initializes a new instance of the class for one selected AV1 operating point. + /// Initializes a new instance of the class for one selected AV1 operating point without a + /// reconstructed reference map. /// /// The zero-based sequence-header operating-point index to decode. public ObuReader(byte operatingPointIndex) => this.operatingPointIndex = operatingPointIndex; + /// + /// Initializes a new instance of the class for one selected AV1 operating point and + /// retained reference map. + /// + /// The zero-based sequence-header operating-point index to decode. + /// The reconstructed reference frames retained by the owning decoder. + public ObuReader(byte operatingPointIndex, Av1ReferenceFrameStore referenceFrames) + { + this.operatingPointIndex = operatingPointIndex; + this.referenceFrames = referenceFrames; + } + /// /// Gets or sets the most recently parsed sequence header. /// @@ -52,14 +130,16 @@ internal class ObuReader public ObuFrameHeader? FrameHeader { get; set; } /// - /// Parses the open bitstream units that make up one coded frame. + /// Parses every open bitstream unit in one bounded AV1 payload. /// /// The reader positioned at the first OBU. - /// The number of bytes available for the coded frame. - /// Creates the tile reader when the first tile payload is encountered. + /// The number of bytes available for the bounded payload. + /// Creates one tile reader when the first tile payload of each coded frame is encountered. /// A value indicating whether each OBU is prefixed by an Annex B length field. public void ReadAll(ref Av1BitStreamReader reader, int dataSize, Func creator, bool isAnnexB = false) { + bool completed = false; + try { int availableByteCount = reader.Length - Av1Math.DivideBy8Floor(reader.BitPosition); @@ -69,9 +149,10 @@ internal class ObuReader } bool seenFrameHeader = false; - bool frameDecodingFinished = false; + int nextTileStart = 0; Span primaryFrameHeaderPayload = default; - while (dataSize > 0 && !frameDecodingFinished) + + while (dataSize > 0) { int annexObuSize = 0; if (isAnnexB) @@ -139,11 +220,17 @@ internal class ObuReader } Av1BitStreamReader payloadReader = new(obuPayload); + bool frameDecodingFinished = false; int decodedPayloadSize; switch (header.Type) { case ObuType.SequenceHeader: + if (seenFrameHeader) + { + throw new InvalidImageContentException("An AV1 sequence header interrupts an incomplete coded frame."); + } + this.SequenceHeader = new(); ReadSequenceHeader(ref payloadReader, this.SequenceHeader); if (this.operatingPointIndex >= this.SequenceHeader.OperatingPoint.Length) @@ -154,6 +241,12 @@ internal class ObuReader } this.currentOperatingPointIdc = this.SequenceHeader.OperatingPoint[this.operatingPointIndex].Idc; + + // A sequence header starts a new reference domain. Clear both the syntax snapshot and decoded + // owners only after the complete header and selected operating point have been accepted, so a + // later inter header cannot pair an empty parser map with samples retained from the old sequence. + this.frameReferenceState.Reset(); + this.referenceFrames?.Reset(); decodedPayloadSize = Av1Math.DivideBy8Floor(payloadReader.BitPosition); break; case ObuType.FrameHeader: @@ -168,7 +261,14 @@ internal class ObuReader } seenFrameHeader = true; - this.FrameHeader = new(); + ObuFrameHeader primaryFrameHeader = new() + { + TemporalId = header.TemporalId, + SpatialId = header.SpatialId + }; + + this.frameReferenceState.InitializeFrameHeader(primaryFrameHeader); + this.FrameHeader = primaryFrameHeader; this.ReadFrameHeader(ref payloadReader, header, trailingBit: true); decodedPayloadSize = Av1Math.DivideBy8Floor(payloadReader.BitPosition); primaryFrameHeaderPayload = obuPayload[..decodedPayloadSize]; @@ -201,7 +301,14 @@ internal class ObuReader } seenFrameHeader = true; - this.FrameHeader = new(); + ObuFrameHeader combinedFrameHeader = new() + { + TemporalId = header.TemporalId, + SpatialId = header.SpatialId + }; + + this.frameReferenceState.InitializeFrameHeader(combinedFrameHeader); + this.FrameHeader = combinedFrameHeader; this.ReadFrameHeader(ref payloadReader, header, trailingBit: false); primaryFrameHeaderPayload = obuPayload[..Av1Math.DivideBy8Floor(payloadReader.BitPosition)]; goto TILE_GROUP; @@ -216,18 +323,17 @@ internal class ObuReader // A combined frame OBU reaches this label after its frame-header portion has // been consumed, leaving the same tile-group syntax as a standalone tile OBU. - this.ReadTileGroup(ref payloadReader, this.decoder, header, out frameDecodingFinished); + this.ReadTileGroup(ref payloadReader, this.decoder, header, ref nextTileStart, out frameDecodingFinished); decodedPayloadSize = Av1Math.DivideBy8Floor(payloadReader.BitPosition); - if (frameDecodingFinished) + break; + case ObuType.TemporalDelimiter: + if (seenFrameHeader) { - seenFrameHeader = false; + throw new InvalidImageContentException("An AV1 temporal delimiter interrupts an incomplete coded frame."); } - break; - case ObuType.TemporalDelimiter: // AV1 section 5.6 defines no delimiter syntax. The common post-switch validation still permits // zero bytes between the empty syntax and the declared payload boundary, matching libaom. - seenFrameHeader = false; decodedPayloadSize = 0; break; case ObuType.Padding: @@ -282,7 +388,26 @@ internal class ObuReader throw new InvalidImageContentException("The AV1 OBU contains nonzero data after its decoded syntax."); } } + + if (frameDecodingFinished) + { + // Complete reconstruction and reference-buffer ownership before publishing the matching syntax + // state. Any decoder failure leaves the preceding session snapshot intact for deterministic cleanup. + this.decoder!.CompleteFrame(); + this.frameReferenceState.CompleteFrame(this.FrameHeader!, this.SequenceHeader!.IsFrameIdNumbersPresent); + this.decoder = null; + seenFrameHeader = false; + nextTileStart = 0; + primaryFrameHeaderPayload = default; + } + } + + if (seenFrameHeader || this.decoder is not null) + { + throw new InvalidImageContentException("The AV1 payload ends before the current coded frame is complete."); } + + completed = true; } catch (IndexOutOfRangeException exception) { @@ -292,6 +417,28 @@ internal class ObuReader { throw new InvalidImageContentException("The AV1 OBU syntax exceeds its payload boundary.", exception); } + finally + { + if (!completed) + { + // A bounded payload can commit earlier layers before a later OBU fails. Those transitions cannot be + // rolled back after displaced owners have been released, so invalidate the complete decoder session. + this.Reset(); + } + } + } + + /// + /// Clears all parser, tile-reader, syntax-reference, and reconstructed-reference state owned by this session. + /// + public void Reset() + { + this.decoder = null; + this.SequenceHeader = null; + this.FrameHeader = null; + this.currentOperatingPointIdc = 0; + this.frameReferenceState.Reset(); + this.referenceFrames?.Reset(); } /// @@ -791,6 +938,7 @@ internal class ObuReader ObuSequenceHeader sequenceHeader = this.SequenceHeader!; ObuFrameHeader frameHeader = this.FrameHeader!; bool useSuperResolution = false; + if (sequenceHeader.EnableSuperResolution) { useSuperResolution = reader.ReadBoolean(); @@ -798,7 +946,8 @@ internal class ObuReader if (useSuperResolution) { - frameHeader.FrameSize.SuperResolutionDenominator = (int)reader.ReadLiteral(Av1Constants.SuperResolutionScaleBits) + Av1Constants.SuperResolutionScaleDenominatorMinimum; + frameHeader.FrameSize.SuperResolutionDenominator = + (int)reader.ReadLiteral(Av1Constants.SuperResolutionScaleBits) + Av1Constants.SuperResolutionScaleDenominatorMinimum; } else { @@ -807,8 +956,8 @@ internal class ObuReader frameHeader.FrameSize.SuperResolutionUpscaledWidth = frameHeader.FrameSize.FrameWidth; - // AV1 signals the upscaled width first. Tile and block decoding use the rounded-down - // coded width obtained from the fixed scale numerator and signaled denominator. + // AV1 signals the upscaled width first. Tile and block decoding use the nearest-integer coded width obtained + // from the fixed scale numerator and signaled denominator. frameHeader.FrameSize.FrameWidth = ((frameHeader.FrameSize.SuperResolutionUpscaledWidth * Av1Constants.ScaleNumerator) + (frameHeader.FrameSize.SuperResolutionDenominator / 2)) / @@ -831,8 +980,11 @@ internal class ObuReader { ObuFrameHeader frameHeader = this.FrameHeader!; bool renderSizeAndFrameSizeDifferent = reader.ReadBoolean(); + if (renderSizeAndFrameSizeDifferent) { + // render_width_minus_1 and render_height_minus_1 are fixed 16-bit fields, independent of the sequence's + // coded-dimension bit widths. frameHeader.FrameSize.RenderWidth = (int)reader.ReadLiteral(16) + 1; frameHeader.FrameSize.RenderHeight = (int)reader.ReadLiteral(16) + 1; } @@ -852,10 +1004,20 @@ internal class ObuReader { ObuSequenceHeader sequenceHeader = this.SequenceHeader!; ObuFrameHeader frameHeader = this.FrameHeader!; + if (frameSizeOverrideFlag) { frameHeader.FrameSize.FrameWidth = (int)reader.ReadLiteral(sequenceHeader.FrameWidthBits) + 1; frameHeader.FrameSize.FrameHeight = (int)reader.ReadLiteral(sequenceHeader.FrameHeightBits) + 1; + + // Section 5.9.7 signals frame dimensions using the sequence maxima's bit widths, but the resulting values + // remain constrained by those maxima. Rejecting the oversized result here prevents later buffer geometry + // from accepting a value that the sequence header does not permit. + if (frameHeader.FrameSize.FrameWidth > sequenceHeader.MaxFrameWidth || + frameHeader.FrameSize.FrameHeight > sequenceHeader.MaxFrameHeight) + { + throw new InvalidImageContentException("AV1 frame dimensions exceed the sequence maximum dimensions."); + } } else { @@ -867,6 +1029,103 @@ internal class ObuReader this.ComputeImageSize(sequenceHeader); } + /// + /// Reads or inherits inter-frame dimensions using the seven selected reference roles. + /// + /// The reader positioned at the frame-size-with-references syntax. + /// The retained reconstructed frames selected by the current reference mapping. + private void ReadFrameSizeWithReferences(ref Av1BitStreamReader reader, Av1ReferenceFrameStore referenceFrames) + { + ObuSequenceHeader sequenceHeader = this.SequenceHeader!; + ObuFrameHeader frameHeader = this.FrameHeader!; + ObuFrameSize frameSize = frameHeader.FrameSize; + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + bool foundReference = false; + + // frame_size_with_refs carries one found_ref bit per selected role only until the first one is set. A set bit + // terminates this syntax immediately; no flags for the remaining roles are present in the bitstream. + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + if (!reader.ReadBoolean()) + { + continue; + } + + Av1ReferenceFrame referenceFrame = referenceFrames.Resolve((int)referenceFrameIndices[reference])!; + ObuFrameSize referenceSize = referenceFrame.FrameHeader.FrameSize; + + // AV1 5.9.7 inherits the reference buffer's visible post-super-resolution dimensions, corresponding to + // libaom's y_crop_width and y_crop_height, plus its render rectangle. The current frame then signals its own + // super-resolution denominator, so the reference's coded width and denominator are not copied. + frameSize.FrameWidth = referenceFrame.FrameBuffer.Width; + frameSize.FrameHeight = referenceFrame.FrameBuffer.Height; + frameSize.RenderWidth = referenceSize.RenderWidth; + frameSize.RenderHeight = referenceSize.RenderHeight; + this.ReadSuperResolutionParameters(ref reader); + this.ComputeImageSize(sequenceHeader); + foundReference = true; + break; + } + + if (!foundReference) + { + // When no reference supplies dimensions, frame_size_with_refs carries the ordinary explicit frame size, + // current super-resolution syntax, and render-size syntax in that order. + this.ReadFrameSize(ref reader, true); + this.ReadRenderSize(ref reader); + } + + bool hasCompatibleReferenceSize = false; + ObuColorConfig colorConfig = sequenceHeader.ColorConfig; + + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + Av1ReferenceFrame referenceFrame = referenceFrames.Resolve((int)referenceFrameIndices[reference])!; + int referenceWidth = referenceFrame.FrameBuffer.Width; + int referenceHeight = referenceFrame.FrameBuffer.Height; + + // AV1 6.8.6 permits a reference dimension from one half through sixteen times the current coded + // dimension. setup_frame_size_with_refs requires at least one of the seven selected roles to satisfy both + // axes before the frame may proceed. + hasCompatibleReferenceSize |= + (2 * frameSize.FrameWidth) >= referenceWidth && + (2 * frameSize.FrameHeight) >= referenceHeight && + frameSize.FrameWidth <= (16 * referenceWidth) && + frameSize.FrameHeight <= (16 * referenceHeight); + + ObuColorConfig referenceColorConfig = referenceFrame.FrameBuffer.ColorConfig; + + // Every selected reference participates in the same prediction sample domain. Mixing bit depth or chroma + // subsampling would change sample interpretation and is prohibited even when that role is not selected by + // any block in the current frame. + if (referenceFrame.FrameBuffer.BitDepth != colorConfig.BitDepth || + referenceColorConfig.SubSamplingX != colorConfig.SubSamplingX || + referenceColorConfig.SubSamplingY != colorConfig.SubSamplingY) + { + throw new InvalidImageContentException("An AV1 inter frame selects a reference with an incompatible color format."); + } + } + + if (!hasCompatibleReferenceSize) + { + throw new InvalidImageContentException("An AV1 inter frame has no reference with compatible dimensions."); + } + } + + /// + /// Reads the frame-level interpolation-filter selection. + /// + /// The reader positioned at the interpolation-filter syntax. + /// The fixed filter family or the per-block switchable selection. + private static Av1InterpolationFilter ReadFrameInterpolationFilter(ref Av1BitStreamReader reader) + { + // A leading one omits the two-bit fixed-family field and delegates the choice to each inter block. Otherwise, + // the literal values map directly to regular, smooth, sharp, and bilinear as defined by AV1 6.10.2. + return reader.ReadBoolean() + ? Av1InterpolationFilter.Switchable + : (Av1InterpolationFilter)reader.ReadLiteral(2); + } + /// /// Reads the tile layout and derives tile boundaries in mode-information units. /// @@ -1027,7 +1286,7 @@ internal class ObuReader } /// - /// Reads the uncompressed syntax for the current still-image frame. + /// Reads the uncompressed syntax for one coded frame in a bounded AV1 image item or layered image sequence. /// /// The reader positioned at the uncompressed frame header. /// The OBU header identifying the frame's temporal and spatial layers. @@ -1035,8 +1294,8 @@ internal class ObuReader { ObuSequenceHeader sequenceHeader = this.SequenceHeader!; ObuFrameHeader frameHeader = this.FrameHeader!; + Av1ReferenceFrame? primaryReference = null; int idLength = sequenceHeader.FrameIdLength; - uint previousFrameId = 0; bool frameSizeOverrideFlag = false; if (sequenceHeader.IsFrameIdNumbersPresent) { @@ -1074,8 +1333,8 @@ internal class ObuReader frameHeader.DisplayFrameId = reader.ReadLiteral(idLength); } - // Showing an existing frame requires sequence reference storage. The image-item decoder deliberately - // owns one independently coded picture and therefore rejects this sequence-only operation at its boundary. + // The bounded image-item decoder retains reference state only to reconstruct coded dependent layers. + // show_existing_frame is a presentation-timeline operation and remains outside that image-only scope. throw new InvalidImageContentException("An AV1 image item cannot display a previously decoded frame."); } @@ -1114,7 +1373,6 @@ internal class ObuReader if (frameHeader.FrameType == ObuFrameType.KeyFrame && frameHeader.ShowFrame) { frameHeader.GetReferenceValidity().Clear(); - frameHeader.GetReferenceOrderHints().Clear(); } frameHeader.DisableCdfUpdate = reader.ReadBoolean(); @@ -1148,45 +1406,28 @@ internal class ObuReader frameHeader.ForceIntegerMotionVector = true; } - bool havePreviousFrameId = !(frameHeader.FrameType == ObuFrameType.KeyFrame && frameHeader.ShowFrame); - if (havePreviousFrameId) - { - previousFrameId = frameHeader.CurrentFrameId; - } + bool havePreviousFrameId = this.frameReferenceState.HasCurrentFrameId && + !(frameHeader.FrameType == ObuFrameType.KeyFrame && frameHeader.ShowFrame); + + uint previousFrameId = this.frameReferenceState.CurrentFrameId; if (sequenceHeader.IsFrameIdNumbersPresent) { frameHeader.CurrentFrameId = reader.ReadLiteral(idLength); if (havePreviousFrameId) { - uint diffFrameId = (frameHeader.CurrentFrameId > previousFrameId) ? - frameHeader.CurrentFrameId - previousFrameId : - (uint)((1 << idLength) + (int)frameHeader.CurrentFrameId - previousFrameId); - if (frameHeader.CurrentFrameId == previousFrameId || diffFrameId >= 1 << (idLength - 1)) + uint frameIdModulus = 1U << idLength; + uint diffFrameId = frameHeader.CurrentFrameId > previousFrameId + ? frameHeader.CurrentFrameId - previousFrameId + : frameIdModulus + frameHeader.CurrentFrameId - previousFrameId; + + if (frameHeader.CurrentFrameId == previousFrameId || diffFrameId >= 1U << (idLength - 1)) { throw new ImageFormatException("Current frame ID cannot be same as previous Frame ID"); } } - int diffLength = sequenceHeader.DeltaFrameIdLength; - Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); - Span referenceValidity = frameHeader.GetReferenceValidity(); - for (int i = 0; i < Av1Constants.ReferenceFrameCount; i++) - { - if (frameHeader.CurrentFrameId > (1U << diffLength)) - { - if ((referenceFrameIndices[i] > frameHeader.CurrentFrameId) || - referenceFrameIndices[i] > (frameHeader.CurrentFrameId - (1 - diffLength))) - { - referenceValidity[i] = false; - } - } - else if (referenceFrameIndices[i] > frameHeader.CurrentFrameId && - referenceFrameIndices[i] < ((1 << idLength) + (frameHeader.CurrentFrameId - (1 << diffLength)))) - { - referenceValidity[i] = false; - } - } + frameHeader.MarkReferenceFrames(idLength, sequenceHeader.DeltaFrameIdLength); } else { @@ -1258,13 +1499,13 @@ internal class ObuReader if (!frameHeader.IsIntra || (frameHeader.RefreshFrameFlags != 0xFFU)) { - if (frameHeader.ErrorResilientMode && sequenceHeader.OrderHintInfo != null) + if (frameHeader.ErrorResilientMode && sequenceHeader.OrderHintInfo.EnableOrderHint) { Span referenceOrderHints = frameHeader.GetReferenceOrderHints(); Span referenceValidity = frameHeader.GetReferenceValidity(); for (int i = 0; i < Av1Constants.ReferenceFrameCount; i++) { - int referenceOrderHint = (int)reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); + uint referenceOrderHint = reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); if (referenceOrderHint != referenceOrderHints[i]) { referenceValidity[i] = false; @@ -1287,8 +1528,54 @@ internal class ObuReader } else { - // Single image is always Intra. - throw new InvalidImageContentException("AVIF image can only contain INTRA frames."); + Av1ReferenceFrameStore? retainedReferenceFrames = this.referenceFrames; + + if (retainedReferenceFrames is null) + { + // Inter-frame size syntax reads dimensions from reconstructed references. Header-only parser users do + // not own those samples, while the production decoder establishes this dependency in its constructor. + throw new InvalidOperationException("AV1 inter-frame parsing requires a reconstructed reference map."); + } + + ReadReferenceFrameIndices(ref reader, sequenceHeader, frameHeader, retainedReferenceFrames); + + if (frameHeader.PrimaryReferenceSlot.HasValue) + { + // Reference-index parsing validates the resolved slot before publishing it on the header. Retaining + // the owner here keeps every inherited frame state tied to the same normative primary reference. + primaryReference = retainedReferenceFrames.Resolve(frameHeader.PrimaryReferenceSlot.Value)!; + } + + if (!frameHeader.ErrorResilientMode && frameSizeOverrideFlag) + { + this.ReadFrameSizeWithReferences(ref reader, retainedReferenceFrames); + } + else + { + this.ReadFrameSize(ref reader, frameSizeOverrideFlag); + this.ReadRenderSize(ref reader); + } + + if (!frameHeader.ForceIntegerMotionVector) + { + frameHeader.AllowHighPrecisionMotionVector = reader.ReadBoolean(); + } + + frameHeader.InterpolationFilter = ReadFrameInterpolationFilter(ref reader); + frameHeader.IsMotionModeSwitchable = reader.ReadBoolean(); + } + + bool mightAllowReferenceFrameMotionVectors = + !frameHeader.ErrorResilientMode && + sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors && + sequenceHeader.OrderHintInfo.EnableOrderHint && + !frameHeader.IsIntra; + + if (mightAllowReferenceFrameMotionVectors) + { + // AV1 5.9.2 carries this flag only when temporal order hints and the sequence-level reference-MV tool are + // both available. All other frame classes derive false without consuming a bit. + frameHeader.UseReferenceFrameMotionVectors = reader.ReadBoolean(); } // SetupFrameBufferReferences(sequenceHeader, frameHeader); @@ -1304,42 +1591,27 @@ internal class ObuReader frameHeader.DisableFrameEndUpdateCdf = reader.ReadBoolean(); } - if (frameHeader.PrimaryReferenceFrame == Av1Constants.PrimaryReferenceFrameNone) - { - // InitConCoefficientCdfs(); - // SetupPastIndependence(frameHeader); - } - else + if (primaryReference is not null) { - // LoadCdfs(frameHeader.PrimaryReferenceFrame); - // LoadPrevious(); - throw new NotImplementedException(); + // When update flags omit new values, loop-filter deltas inherit from the primary frame. Copying the two + // fixed tables before parsing lets the existing header object retain unchanged entries without aliases. + primaryReference.FrameHeader.LoopFilterParameters.ReferenceDeltas.AsSpan().CopyTo(frameHeader.LoopFilterParameters.ReferenceDeltas); + primaryReference.FrameHeader.LoopFilterParameters.ModeDeltas.AsSpan().CopyTo(frameHeader.LoopFilterParameters.ModeDeltas); } - if (frameHeader.UseReferenceFrameMotionVectors) - { - // MotionFieldEstimations(); - throw new NotImplementedException(); - } + // Entropy defaults depend on base_q_idx, which follows tile information in the header. Av1TileReader therefore + // loads either the retained primary snapshot or the selected quantizer-band defaults at the first tile boundary. // GenerateNextReferenceFrameMap(sequenceHeader, frameHeader); frameHeader.TilesInfo = ReadTileInfo(ref reader, sequenceHeader, frameHeader); ReadQuantizationParameters(ref reader, sequenceHeader, frameHeader); - ReadSegmentationParameters(ref reader, frameHeader); + ReadSegmentationParameters(ref reader, frameHeader, primaryReference?.FrameHeader.SegmentationParameters); ReadFrameDeltaQParameters(ref reader, frameHeader); ReadFrameDeltaLoopFilterParameters(ref reader, frameHeader); // SetupSegmentationDequantization(); - if (frameHeader.PrimaryReferenceFrame == Av1Constants.PrimaryReferenceFrameNone) - { - // ResetParseContext(mainParseContext, frameHeader.QuantizationParameters.BaseQIndex); - } - else - { - // LoadPreviousSegmentIds(); - throw new NotImplementedException(); - } - + // The primary frame retains its decoded segment map in Av1FrameInfo. Inter block parsing copies or predicts + // segment identifiers from that map according to update_map instead of duplicating it in the frame header. Av1QuantizationLookup.UpdateFrameQuantizationState(frameHeader); if (frameHeader.CodedLossless) @@ -1364,8 +1636,91 @@ internal class ObuReader } frameHeader.UseReducedTransformSet = reader.ReadBoolean(); - ReadGlobalMotionParameters(ref reader, sequenceHeader, frameHeader); - frameHeader.FilmGrainParameters = ReadFilmGrainFilterParameters(ref reader, sequenceHeader, frameHeader); + this.ReadGlobalMotionParameters(ref reader, frameHeader); + this.ReadFilmGrainFilterParameters(ref reader, sequenceHeader, frameHeader); + } + + /// + /// Reads or derives the seven reference-map slots used by an inter frame and resolves its primary context source. + /// + /// The reader positioned at the inter-reference signaling syntax. + /// The sequence header defining frame-ID and order-hint domains. + /// The frame header that receives the seven-entry reference mapping. + /// The retained reconstructed frames backing the eight reference-map slots. + private static void ReadReferenceFrameIndices( + ref Av1BitStreamReader reader, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1ReferenceFrameStore referenceFrames) + { + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + Span referenceValidity = frameHeader.GetReferenceValidity(); + bool usesShortSignaling = sequenceHeader.OrderHintInfo.EnableOrderHint && reader.ReadBoolean(); + + if (usesShortSignaling) + { + uint lastFrameIndex = reader.ReadLiteral(ReferenceFrameIndexBits); + uint goldenFrameIndex = reader.ReadLiteral(ReferenceFrameIndexBits); + InlineArray8 slotOccupancyStorage = default; + Span slotOccupancy = slotOccupancyStorage; + + referenceFrames.FillOccupancy(slotOccupancy); + + // Short signaling transmits only LAST and GOLDEN. The remaining five roles are a normative derivation from + // the persisted slot order hints and physical slot occupancy, not frame-ID validity or a decoder heuristic. + Av1ReferenceFrameDerivation.DeriveShortSignaledReferences( + frameHeader.OrderHint, + sequenceHeader.OrderHintInfo.OrderHintBits, + lastFrameIndex, + goldenFrameIndex, + frameHeader.GetReferenceOrderHints(), + slotOccupancy, + referenceFrameIndices); + } + + Span referenceFrameIds = frameHeader.GetReferenceFrameIds(); + uint frameIdModulus = 1U << sequenceHeader.FrameIdLength; + + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + uint slot = referenceFrameIndices[reference]; + if (!usesShortSignaling) + { + slot = reader.ReadLiteral(ReferenceFrameIndexBits); + referenceFrameIndices[reference] = slot; + } + + // Slot occupancy and frame-ID validity are independent normative states. Short signaling derives roles + // from every occupied slot before this per-role validity check, matching av1_set_frame_refs followed by + // libaom's valid_for_referencing check. + if (referenceFrames.Resolve((int)slot) is null) + { + throw new InvalidImageContentException("An AV1 inter frame selects an unoccupied reference-map slot."); + } + + if (!referenceValidity[(int)slot]) + { + throw new InvalidImageContentException("An AV1 inter frame selects a reference that is not valid for referencing."); + } + + if (sequenceHeader.IsFrameIdNumbersPresent) + { + uint deltaFrameId = reader.ReadLiteral(sequenceHeader.DeltaFrameIdLength) + 1U; + uint expectedFrameId = (frameHeader.CurrentFrameId + frameIdModulus - deltaFrameId) % frameIdModulus; + + if (referenceFrameIds[(int)slot] != expectedFrameId) + { + throw new InvalidImageContentException("An AV1 inter reference does not match its signaled frame identifier."); + } + } + } + + if (frameHeader.PrimaryReferenceFrame != Av1Constants.PrimaryReferenceFrameNone) + { + // primary_ref_frame indexes the seven inter-reference roles, not the eight-slot retained map. Resolve it + // once so entropy, segmentation, loop-filter, and motion state all select the same retained owner later. + frameHeader.PrimaryReferenceSlot = (byte)referenceFrameIndices[(int)frameHeader.PrimaryReferenceFrame]; + } } /// @@ -1406,8 +1761,14 @@ internal class ObuReader /// The reader positioned at the tile-group payload. /// The tile reader that decodes each tile payload. /// The OBU header containing the remaining tile-group payload size. - /// Receives whether this group contains the final tile of the frame. - private void ReadTileGroup(ref Av1BitStreamReader reader, IAv1TileReader decoder, ObuHeader header, out bool isLastTileGroup) + /// The zero-based tile index that must begin this group and receives the next expected index. + /// Receives whether this group completes the frame's ordered tile coverage. + private void ReadTileGroup( + ref Av1BitStreamReader reader, + IAv1TileReader decoder, + ObuHeader header, + ref int nextTileStart, + out bool isLastTileGroup) { ObuSequenceHeader sequenceHeader = this.SequenceHeader!; ObuFrameHeader frameHeader = this.FrameHeader!; @@ -1420,9 +1781,9 @@ internal class ObuReader tileStartAndEndPresentFlag = reader.ReadBoolean(); } - if (header.Type == ObuType.FrameHeader) + if (header.Type == ObuType.Frame && tileStartAndEndPresentFlag) { - DebugGuard.IsFalse(tileStartAndEndPresentFlag, nameof(tileStartAndEndPresentFlag), "Frame header should not set 'tileStartAndEndPresentFlag'."); + throw new InvalidImageContentException("A combined AV1 frame OBU cannot signal explicit tile-group bounds."); } int tileGroupStart = 0; @@ -1434,7 +1795,11 @@ internal class ObuReader tileGroupEnd = (int)reader.ReadLiteral(tileBits); } - isLastTileGroup = (tileGroupEnd + 1) == tileCount; + if (tileGroupStart != nextTileStart || tileGroupStart > tileGroupEnd || tileGroupEnd >= tileCount) + { + throw new InvalidImageContentException("The AV1 tile groups do not provide complete ordered frame coverage."); + } + AlignToByteBoundary(ref reader); int endBitPosition = reader.BitPosition; int headerBytes = (endBitPosition - startBitPosition) / 8; @@ -1467,6 +1832,9 @@ internal class ObuReader decoder.ReadTile(tileData, tileNum); } + nextTileStart = tileGroupEnd + 1; + isLastTileGroup = nextTileStart == tileCount; + if (tileGroupEnd != tileCount - 1) { return; @@ -1605,7 +1973,13 @@ internal class ObuReader /// /// The reader positioned at the segmentation parameters. /// The frame header that receives the segmentation state. - private static void ReadSegmentationParameters(ref Av1BitStreamReader reader, ObuFrameHeader frameHeader) + /// + /// The primary-reference feature state, or when the frame has no primary reference. + /// + private static void ReadSegmentationParameters( + ref Av1BitStreamReader reader, + ObuFrameHeader frameHeader, + ObuSegmentationParameters? primaryParameters) { frameHeader.SegmentationParameters.Enabled = reader.ReadBoolean(); @@ -1658,6 +2032,12 @@ internal class ObuReader } } } + else + { + // update_data equal to zero preserves the complete feature mask and values from the primary frame. + // The current header owns its arrays, so later reference replacement cannot mutate inherited state. + frameHeader.SegmentationParameters.CopyFeaturesFrom(primaryParameters!); + } } else { @@ -1860,20 +2240,139 @@ internal class ObuReader /// Reads global-motion parameters when permitted by the frame type. /// /// The reader positioned at the global-motion parameters. - /// The sequence header controlling global-motion tools. /// The current frame header. - private static void ReadGlobalMotionParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + private void ReadGlobalMotionParameters(ref Av1BitStreamReader reader, ObuFrameHeader frameHeader) { - _ = reader; - _ = sequenceHeader; + Span parameters = frameHeader.GetGlobalMotionParameters(); + parameters.Fill(Av1GlobalMotionParameters.Identity); if (frameHeader.IsIntra) { return; } - // Not applicable for INTRA frames. - throw new NotImplementedException(); + ObuFrameHeader? primaryReferenceHeader = null; + byte? primaryReferenceSlot = frameHeader.PrimaryReferenceSlot; + if (primaryReferenceSlot is not null) + { + // primary_ref_frame identifies the preceding frame whose same seven canonical reference roles supply the + // recentering values. Reference-slot validation has already completed before this syntax is reached. + primaryReferenceHeader = this.referenceFrames!.Resolve(primaryReferenceSlot.Value)!.FrameHeader; + } + + for (int referenceIndex = 0; referenceIndex < Av1Constants.ReferencesPerFrame; referenceIndex++) + { + Av1GlobalMotionParameters referenceParameters = primaryReferenceHeader is null + ? Av1GlobalMotionParameters.Identity + : primaryReferenceHeader.GetGlobalMotionParameters()[referenceIndex]; + + ReadGlobalMotionModel( + ref reader, + ref parameters[referenceIndex], + referenceParameters, + frameHeader.AllowHighPrecisionMotionVector); + } + } + + /// + /// Reads one global-motion model relative to the corresponding model retained by the primary reference frame. + /// + /// The reader positioned at the model type and parameter syntax. + /// The destination global-motion model. + /// The same-role model retained by the primary reference frame. + /// + /// A value indicating whether translation-only parameters retain their high-precision bit. + /// + private static void ReadGlobalMotionModel( + ref Av1BitStreamReader reader, + ref Av1GlobalMotionParameters parameters, + Av1GlobalMotionParameters referenceParameters, + bool allowHighPrecisionMotionVector) + { + Av1GlobalMotionType type = Av1GlobalMotionType.Identity; + if (reader.ReadBoolean()) + { + if (reader.ReadBoolean()) + { + type = Av1GlobalMotionType.RotationZoom; + } + else + { + type = reader.ReadBoolean() ? Av1GlobalMotionType.Translation : Av1GlobalMotionType.Affine; + } + } + + parameters = Av1GlobalMotionParameters.Identity; + parameters.Type = type; + if (type >= Av1GlobalMotionType.RotationZoom) + { + // Diagonal terms are coded as a delta from the identity scale, whereas off-diagonal terms are centered + // directly around zero. Both are restored to the common sixteen-bit matrix precision after decoding. + parameters[2] = + (reader.ReadSignedReferenceSubexponential( + GlobalMotionAlphaValueMagnitude, + GlobalMotionSubexponentialGroupBitCount, + (referenceParameters[2] >> GlobalMotionAlphaPrecisionDifference) - (1 << GlobalMotionAlphaPrecisionBits)) * GlobalMotionAlphaDecodeFactor) + + Av1GlobalMotionParameters.ModelScale; + + parameters[3] = reader.ReadSignedReferenceSubexponential( + GlobalMotionAlphaValueMagnitude, + GlobalMotionSubexponentialGroupBitCount, + referenceParameters[3] >> GlobalMotionAlphaPrecisionDifference) * GlobalMotionAlphaDecodeFactor; + } + + if (type >= Av1GlobalMotionType.Affine) + { + parameters[4] = reader.ReadSignedReferenceSubexponential( + GlobalMotionAlphaValueMagnitude, + GlobalMotionSubexponentialGroupBitCount, + referenceParameters[4] >> GlobalMotionAlphaPrecisionDifference) * GlobalMotionAlphaDecodeFactor; + + parameters[5] = + (reader.ReadSignedReferenceSubexponential( + GlobalMotionAlphaValueMagnitude, + GlobalMotionSubexponentialGroupBitCount, + (referenceParameters[5] >> GlobalMotionAlphaPrecisionDifference) - (1 << GlobalMotionAlphaPrecisionBits)) * GlobalMotionAlphaDecodeFactor) + + Av1GlobalMotionParameters.ModelScale; + } + else + { + // Rotation-zoom constrains the second matrix row to the perpendicular vector of the first row. Identity + // and translation models retain the same derived identity coefficients. + parameters[4] = -parameters[3]; + parameters[5] = parameters[2]; + } + + if (type >= Av1GlobalMotionType.Translation) + { + // Translation-only models use a wider coordinate domain than affine models. When high-precision motion is + // disabled, AV1 removes one coded bit and adds one reconstruction shift so the physical displacement grid + // remains in quarter-sample units. Affine translation retains the fixed model-to-translation precision gap. + int precisionAdjustment = type == Av1GlobalMotionType.Translation && !allowHighPrecisionMotionVector ? 1 : 0; + int translationBits = type == Av1GlobalMotionType.Translation + ? GlobalMotionAbsoluteTranslationOnlyBits - precisionAdjustment + : GlobalMotionAbsoluteTranslationBits; + + int translationPrecisionDifference = type == Av1GlobalMotionType.Translation + ? Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionTranslationOnlyPrecisionBits + precisionAdjustment + : Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionTranslationPrecisionBits; + + int translationDecodeFactor = 1 << translationPrecisionDifference; + int translationValueMagnitude = (1 << translationBits) + 1; + parameters[0] = reader.ReadSignedReferenceSubexponential( + translationValueMagnitude, + GlobalMotionSubexponentialGroupBitCount, + referenceParameters[0] >> translationPrecisionDifference) * translationDecodeFactor; + + parameters[1] = reader.ReadSignedReferenceSubexponential( + translationValueMagnitude, + GlobalMotionSubexponentialGroupBitCount, + referenceParameters[1] >> translationPrecisionDifference) * translationDecodeFactor; + } + + // Invalid shear does not invalidate the frame header. AV1 retains the decoded model and marks it unavailable + // to warped prediction, which is why validity is stored with the parameters instead of throwing here. + parameters.UpdateShearParameters(); } /// @@ -1925,19 +2424,18 @@ internal class ObuReader /// The reader positioned at the film-grain parameters. /// The sequence header defining film-grain availability and color sampling. /// The current frame header. - /// The parsed film-grain parameters. - private static ObuFilmGrainParameters ReadFilmGrainFilterParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + private void ReadFilmGrainFilterParameters(ref Av1BitStreamReader reader, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) { - ObuFilmGrainParameters grainParams = new(); + ObuFilmGrainParameters grainParams = frameHeader.FilmGrainParameters; if (!sequenceHeader.AreFilmGrainingParametersPresent || (!frameHeader.ShowFrame && !frameHeader.ShowableFrame)) { - return grainParams; + return; } grainParams.ApplyGrain = reader.ReadBoolean(); if (!grainParams.ApplyGrain) { - return grainParams; + return; } grainParams.GrainSeed = reader.ReadLiteral(16); @@ -1948,29 +2446,56 @@ internal class ObuReader } else { - // Only inter frames can inherit parameters from a reference frame. Still-image - // intra frames always carry a complete parameter set when grain is enabled. + // Only inter frames can inherit parameters from a reference frame. Intra frames always carry a complete + // parameter set when grain is enabled. grainParams.UpdateGrain = true; } if (!grainParams.UpdateGrain) { - grainParams.FilmGrainParamsRefidx = reader.ReadLiteral(3); - uint tempGrainSeed = grainParams.GrainSeed; + grainParams.FilmGrainParamsRefIdx = reader.ReadLiteral(3); + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + bool isSelectedReference = false; + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + isSelectedReference |= referenceFrameIndices[reference] == grainParams.FilmGrainParamsRefIdx; + } + + Av1ReferenceFrame? referenceFrame = isSelectedReference + ? this.referenceFrames?.Resolve((int)grainParams.FilmGrainParamsRefIdx) + : null; + + if (referenceFrame is null) + { + throw new InvalidImageContentException("The AV1 film-grain reference does not provide retained parameters."); + } - // TODO: implement load_grain_params - // load_grain_params(film_grain_params_ref_idx) - grainParams.GrainSeed = tempGrainSeed; - return grainParams; + uint grainSeed = grainParams.GrainSeed; + uint referenceIndex = grainParams.FilmGrainParamsRefIdx; + + // AV1 inherits the complete parameter set but always uses the new frame's independently signaled seed. + // Fixed inline buffers make this a value copy rather than nine small array allocations. + grainParams.CopyFrom(referenceFrame.FrameHeader.FilmGrainParameters); + grainParams.GrainSeed = grainSeed; + grainParams.FilmGrainParamsRefIdx = referenceIndex; + return; } grainParams.NumYPoints = reader.ReadLiteral(4); - grainParams.PointYValue = new uint[grainParams.NumYPoints]; - grainParams.PointYScaling = new uint[grainParams.NumYPoints]; + if (grainParams.NumYPoints > 14) + { + throw new InvalidImageContentException("The AV1 film-grain luma scaling function exceeds fourteen points."); + } + for (int i = 0; i < grainParams.NumYPoints; i++) { - grainParams.PointYValue[i] = reader.ReadLiteral(8); - grainParams.PointYScaling[i] = reader.ReadLiteral(8); + grainParams.PointYValue[i] = (byte)reader.ReadLiteral(8); + if (i > 0 && grainParams.PointYValue[i] <= grainParams.PointYValue[i - 1]) + { + throw new InvalidImageContentException("The AV1 film-grain luma scaling coordinates are not strictly increasing."); + } + + grainParams.PointYScaling[i] = (byte)reader.ReadLiteral(8); } if (sequenceHeader.ColorConfig.IsMonochrome) @@ -1992,21 +2517,44 @@ internal class ObuReader else { grainParams.NumCbPoints = reader.ReadLiteral(4); - grainParams.PointCbValue = new uint[grainParams.NumCbPoints]; - grainParams.PointCbScaling = new uint[grainParams.NumCbPoints]; + if (grainParams.NumCbPoints > 10) + { + throw new InvalidImageContentException("The AV1 film-grain blue-difference scaling function exceeds ten points."); + } + for (int i = 0; i < grainParams.NumCbPoints; i++) { - grainParams.PointCbValue[i] = reader.ReadLiteral(8); - grainParams.PointCbScaling[i] = reader.ReadLiteral(8); + grainParams.PointCbValue[i] = (byte)reader.ReadLiteral(8); + if (i > 0 && grainParams.PointCbValue[i] <= grainParams.PointCbValue[i - 1]) + { + throw new InvalidImageContentException("The AV1 film-grain blue-difference scaling coordinates are not strictly increasing."); + } + + grainParams.PointCbScaling[i] = (byte)reader.ReadLiteral(8); } grainParams.NumCrPoints = reader.ReadLiteral(4); - grainParams.PointCrValue = new uint[grainParams.NumCrPoints]; - grainParams.PointCrScaling = new uint[grainParams.NumCrPoints]; + if (grainParams.NumCrPoints > 10) + { + throw new InvalidImageContentException("The AV1 film-grain red-difference scaling function exceeds ten points."); + } + for (int i = 0; i < grainParams.NumCrPoints; i++) { - grainParams.PointCrValue[i] = reader.ReadLiteral(8); - grainParams.PointCrScaling[i] = reader.ReadLiteral(8); + grainParams.PointCrValue[i] = (byte)reader.ReadLiteral(8); + if (i > 0 && grainParams.PointCrValue[i] <= grainParams.PointCrValue[i - 1]) + { + throw new InvalidImageContentException("The AV1 film-grain red-difference scaling coordinates are not strictly increasing."); + } + + grainParams.PointCrScaling[i] = (byte)reader.ReadLiteral(8); + } + + if (sequenceHeader.ColorConfig.SubSamplingX && + sequenceHeader.ColorConfig.SubSamplingY && + (grainParams.NumCbPoints == 0) != (grainParams.NumCrPoints == 0)) + { + throw new InvalidImageContentException("AV1 4:2:0 film grain must apply to both chroma planes or neither."); } } @@ -2018,10 +2566,9 @@ internal class ObuReader if (grainParams.NumYPoints != 0) { numPosChroma = numPosLuma + 1; - grainParams.ArCoeffsYPlus128 = new uint[numPosLuma]; for (int i = 0; i < numPosLuma; i++) { - grainParams.ArCoeffsYPlus128[i] = reader.ReadLiteral(8); + grainParams.ArCoeffsYPlus128[i] = (byte)reader.ReadLiteral(8); } } else @@ -2031,19 +2578,17 @@ internal class ObuReader if (grainParams.ChromaScalingFromLuma || grainParams.NumCbPoints != 0) { - grainParams.ArCoeffsCbPlus128 = new uint[numPosChroma]; for (int i = 0; i < numPosChroma; i++) { - grainParams.ArCoeffsCbPlus128[i] = reader.ReadLiteral(8); + grainParams.ArCoeffsCbPlus128[i] = (byte)reader.ReadLiteral(8); } } if (grainParams.ChromaScalingFromLuma || grainParams.NumCrPoints != 0) { - grainParams.ArCoeffsCrPlus128 = new uint[numPosChroma]; for (int i = 0; i < numPosChroma; i++) { - grainParams.ArCoeffsCrPlus128[i] = reader.ReadLiteral(8); + grainParams.ArCoeffsCrPlus128[i] = (byte)reader.ReadLiteral(8); } } @@ -2065,8 +2610,6 @@ internal class ObuReader grainParams.OverlapFlag = reader.ReadBoolean(); grainParams.ClipToRestrictedRange = reader.ReadBoolean(); - - return grainParams; } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs index 0adb53d68..6e74a6185 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSegmentationParameters.cs @@ -59,6 +59,24 @@ internal class ObuSegmentationParameters /// The segment identifier. /// The feature to inspect. /// when the feature is active; otherwise, . - internal bool IsFeatureActive(int segmentId, ObuSegmentationLevelFeature feature) + public bool IsFeatureActive(int segmentId, ObuSegmentationLevelFeature feature) => this.FeatureEnabled[segmentId, (int)feature]; + + /// + /// Replaces every feature enable flag and value with state from a primary reference frame. + /// + /// The primary-reference segmentation state. + public void CopyFeaturesFrom(ObuSegmentationParameters source) + { + // AV1 inherits feature data but not the current frame's enabled or update flags. Both dimensions are fixed by + // the bitstream syntax, and copying values into this header prevents retained frames from sharing mutable state. + for (int segment = 0; segment < Av1Constants.MaxSegmentCount; segment++) + { + for (int feature = 0; feature < Av1Constants.SegmentationLevelMax; feature++) + { + this.FeatureEnabled[segment, feature] = source.FeatureEnabled[segment, feature]; + this.FeatureData[segment, feature] = source.FeatureData[segment, feature]; + } + } + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs index 727dd373f..376b7f108 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs @@ -858,8 +858,6 @@ internal class ObuWriter writer.WriteLiteral(grainParams.GrainSeed, 16); writer.WriteLiteral(grainParams.NumYPoints, 4); - Guard.NotNull(grainParams.PointYValue); - Guard.NotNull(grainParams.PointYScaling); for (int i = 0; i < grainParams.NumYPoints; i++) { writer.WriteLiteral(grainParams.PointYValue[i], 8); @@ -876,8 +874,6 @@ internal class ObuWriter (!sequenceHeader.ColorConfig.SubSamplingX || !sequenceHeader.ColorConfig.SubSamplingY || grainParams.NumYPoints != 0)) { writer.WriteLiteral(grainParams.NumCbPoints, 4); - Guard.NotNull(grainParams.PointCbValue); - Guard.NotNull(grainParams.PointCbScaling); for (int i = 0; i < grainParams.NumCbPoints; i++) { writer.WriteLiteral(grainParams.PointCbValue[i], 8); @@ -885,8 +881,6 @@ internal class ObuWriter } writer.WriteLiteral(grainParams.NumCrPoints, 4); - Guard.NotNull(grainParams.PointCrValue); - Guard.NotNull(grainParams.PointCrScaling); for (int i = 0; i < grainParams.NumCrPoints; i++) { writer.WriteLiteral(grainParams.PointCrValue[i], 8); @@ -902,7 +896,6 @@ internal class ObuWriter if (grainParams.NumYPoints != 0) { numPosChroma++; - Guard.NotNull(grainParams.ArCoeffsYPlus128); for (int i = 0; i < numPosLuma; i++) { writer.WriteLiteral(grainParams.ArCoeffsYPlus128[i], 8); @@ -911,7 +904,6 @@ internal class ObuWriter if (grainParams.ChromaScalingFromLuma || grainParams.NumCbPoints != 0) { - Guard.NotNull(grainParams.ArCoeffsCbPlus128); for (int i = 0; i < numPosChroma; i++) { writer.WriteLiteral(grainParams.ArCoeffsCbPlus128[i], 8); @@ -920,7 +912,6 @@ internal class ObuWriter if (grainParams.ChromaScalingFromLuma || grainParams.NumCrPoints != 0) { - Guard.NotNull(grainParams.ArCoeffsCrPlus128); for (int i = 0; i < numPosChroma; i++) { writer.WriteLiteral(grainParams.ArCoeffsCrPlus128[i], 8); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs index 69d366064..9a95d94d9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs @@ -3,7 +3,6 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; @@ -14,7 +13,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// -/// Reconstructs the coded blocks of one AV1 still-image frame into planar sample buffers. +/// Reconstructs the coded blocks of one AV1 image frame into planar sample buffers. /// internal sealed class Av1FrameDecoder : IAv1FrameDecoder, IDisposable { @@ -132,13 +131,8 @@ internal sealed class Av1FrameDecoder : IAv1FrameDecoder, IDisposable loopRestorationDecoder.DecodeFrame(); } - // Film grain belongs to the displayed image rather than the reference reconstruction, so it - // follows every in-loop filter. This decoder owns no retained reference frames. - Av1FilmGrainDecoder filmGrainDecoder = new(this.sequenceHeader, this.frameHeader, this.frameBuffer); - filmGrainDecoder.DecodeFrame(); - - // Extending reference-frame borders is sequence playback state and is deliberately outside - // this still-image decoder's scope. + // Film grain is deliberately excluded here because this buffer is the normative post-restoration reference. + // The owning decoder applies grain only to the presentation buffer after reference ownership is established. } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainDecoder.cs index 1b4dc5d70..1b5dcfdcb 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainDecoder.cs @@ -907,7 +907,7 @@ internal sealed class Av1FilmGrainDecoder int roundingOffset = 1 << (coefficientShift - 1); int grainMinimum = -(1 << (bitDepth - 1)); int grainMaximum = (1 << (bitDepth - 1)) - 1; - uint[] coefficients = parameters.ArCoeffsYPlus128!; + ReadOnlySpan coefficients = parameters.ArCoeffsYPlus128; // TemplatePadding leaves every lag-one through lag-three predecessor addressable without a boundary branch. // Raster order guarantees that all rows above and all samples to the left have already been filtered. @@ -1005,8 +1005,8 @@ internal sealed class Av1FilmGrainDecoder int roundingOffset = 1 << (coefficientShift - 1); int grainMinimum = -(1 << (bitDepth - 1)); int grainMaximum = (1 << (bitDepth - 1)) - 1; - uint[]? cbCoefficients = parameters.ArCoeffsCbPlus128; - uint[]? crCoefficients = parameters.ArCoeffsCrPlus128; + ReadOnlySpan cbCoefficients = parameters.ArCoeffsCbPlus128; + ReadOnlySpan crCoefficients = parameters.ArCoeffsCrPlus128; // Cb and Cr share the same causal predecessor walk, so both accumulators advance one coefficient index // together. A disabled plane stays zero but does not alter the coefficient ordering of the enabled plane. @@ -1024,12 +1024,12 @@ internal sealed class Av1FilmGrainDecoder int grainIndex = ((row + relativeRow) * stride) + column + relativeColumn; if (applyCb) { - weightedCb += ((int)cbCoefficients![coefficientIndex] - 128) * cbGrain[grainIndex]; + weightedCb += ((int)cbCoefficients[coefficientIndex] - 128) * cbGrain[grainIndex]; } if (applyCr) { - weightedCr += ((int)crCoefficients![coefficientIndex] - 128) * crGrain[grainIndex]; + weightedCr += ((int)crCoefficients[coefficientIndex] - 128) * crGrain[grainIndex]; } coefficientIndex++; @@ -1041,12 +1041,12 @@ internal sealed class Av1FilmGrainDecoder int grainIndex = (row * stride) + column + relativeColumn; if (applyCb) { - weightedCb += ((int)cbCoefficients![coefficientIndex] - 128) * cbGrain[grainIndex]; + weightedCb += ((int)cbCoefficients[coefficientIndex] - 128) * cbGrain[grainIndex]; } if (applyCr) { - weightedCr += ((int)crCoefficients![coefficientIndex] - 128) * crGrain[grainIndex]; + weightedCr += ((int)crCoefficients[coefficientIndex] - 128) * crGrain[grainIndex]; } coefficientIndex++; @@ -1072,12 +1072,12 @@ internal sealed class Av1FilmGrainDecoder averageLuma = (averageLuma + ((1 << averagingShift) >> 1)) >> averagingShift; if (applyCb) { - weightedCb += ((int)cbCoefficients![coefficientIndex] - 128) * averageLuma; + weightedCb += ((int)cbCoefficients[coefficientIndex] - 128) * averageLuma; } if (applyCr) { - weightedCr += ((int)crCoefficients![coefficientIndex] - 128) * averageLuma; + weightedCr += ((int)crCoefficients[coefficientIndex] - 128) * averageLuma; } } @@ -1141,8 +1141,8 @@ internal sealed class Av1FilmGrainDecoder /// The number of populated control points. /// The destination scaling lookup table. private static void InitializeScalingFunction( - uint[]? pointValues, - uint[]? pointScalings, + ReadOnlySpan pointValues, + ReadOnlySpan pointScalings, int pointCount, Span lookup) { @@ -1151,27 +1151,24 @@ internal sealed class Av1FilmGrainDecoder return; } - uint[] values = pointValues!; - uint[] scalings = pointScalings!; - // Values outside the first and last control points extend their nearest endpoint rather than extrapolating. - lookup[..(int)values[0]].Fill((int)scalings[0]); + lookup[..(int)pointValues[0]].Fill((int)pointScalings[0]); for (int point = 0; point < pointCount - 1; point++) { - int deltaY = (int)scalings[point + 1] - (int)scalings[point]; - int deltaX = (int)values[point + 1] - (int)values[point]; + int deltaY = (int)pointScalings[point + 1] - (int)pointScalings[point]; + int deltaX = (int)pointValues[point + 1] - (int)pointValues[point]; // A rounded Q16 reciprocal performs the piecewise-linear interpolation using integer arithmetic. The // 32768 bias below rounds each reconstructed scaling value when it returns to integer precision. long delta = deltaY * ((65536 + (deltaX >> 1)) / deltaX); for (int x = 0; x < deltaX; x++) { - lookup[(int)values[point] + x] = (int)scalings[point] + + lookup[(int)pointValues[point] + x] = (int)pointScalings[point] + (int)(((x * delta) + 32768) >> 16); } } - lookup[(int)values[pointCount - 1]..].Fill((int)scalings[pointCount - 1]); + lookup[(int)pointValues[pointCount - 1]..].Fill((int)pointScalings[pointCount - 1]); } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs index ce1763015..975b6bbe2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs @@ -6,7 +6,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// -/// Defines reconstruction of decoded AV1 superblocks within a single still-image frame. +/// Defines reconstruction of decoded AV1 superblocks within one coded image frame. /// internal interface IAv1FrameDecoder { diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1ChromaPredictionMode.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1ChromaPredictionMode.cs new file mode 100644 index 000000000..c443b86ce --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1ChromaPredictionMode.cs @@ -0,0 +1,90 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Identifies the intra prediction mode used by the chroma planes of an AV1 coding block. +/// +internal enum Av1ChromaPredictionMode : byte +{ + /// + /// Predicts each sample from the average of the available top and left neighbors. + /// + DC, + + /// + /// Repeats the top neighboring row vertically through the block. + /// + Vertical, + + /// + /// Repeats the left neighboring column horizontally through the block. + /// + Horizontal, + + /// + /// Projects neighboring samples into the block at 45 degrees. + /// + Directional45Degrees, + + /// + /// Projects neighboring samples into the block at 135 degrees. + /// + Directional135Degrees, + + /// + /// Projects neighboring samples into the block at 113 degrees. + /// + Directional113Degrees, + + /// + /// Projects neighboring samples into the block at 157 degrees. + /// + Directional157Degrees, + + /// + /// Projects neighboring samples into the block at 203 degrees. + /// + Directional203Degrees, + + /// + /// Projects neighboring samples into the block at 67 degrees. + /// + Directional67Degrees, + + /// + /// Blends horizontal and vertical smooth predictions. + /// + Smooth, + + /// + /// Interpolates vertically between the top row and the bottom-left neighbor. + /// + SmoothVertical, + + /// + /// Interpolates horizontally between the left column and the top-right neighbor. + /// + SmoothHorizontal, + + /// + /// Selects the neighbor with the smallest gradient from the top-left reference. + /// + Paeth, + + /// + /// Predicts chroma from the reconstructed luma AC surface. + /// + ChromaFromLuma, + + /// + /// The exclusive upper bound of valid chroma intra-prediction modes. + /// + ModeCount, + + /// + /// Identifies an unavailable chroma prediction mode on an inter-predicted block. + /// + Invalid, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1ChromaPredictionModeExtensions.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1ChromaPredictionModeExtensions.cs new file mode 100644 index 000000000..389e9daa0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1ChromaPredictionModeExtensions.cs @@ -0,0 +1,46 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Provides luma-equivalent prediction metadata for AV1 chroma intra-prediction modes. +/// +internal static class Av1ChromaPredictionModeExtensions +{ + /// + /// Maps a chroma intra-prediction mode to the equivalent luma intra-prediction mode. + /// + /// The chroma intra-prediction mode. + /// The luma mode with the same spatial predictor, or the invalid luma sentinel for an invalid chroma mode. + public static Av1PredictionMode ToLumaMode(this Av1ChromaPredictionMode mode) + => mode switch + { + Av1ChromaPredictionMode.DC => Av1PredictionMode.DC, + Av1ChromaPredictionMode.Vertical => Av1PredictionMode.Vertical, + Av1ChromaPredictionMode.Horizontal => Av1PredictionMode.Horizontal, + Av1ChromaPredictionMode.Directional45Degrees => Av1PredictionMode.Directional45Degrees, + Av1ChromaPredictionMode.Directional135Degrees => Av1PredictionMode.Directional135Degrees, + Av1ChromaPredictionMode.Directional113Degrees => Av1PredictionMode.Directional113Degrees, + Av1ChromaPredictionMode.Directional157Degrees => Av1PredictionMode.Directional157Degrees, + Av1ChromaPredictionMode.Directional203Degrees => Av1PredictionMode.Directional203Degrees, + Av1ChromaPredictionMode.Directional67Degrees => Av1PredictionMode.Directional67Degrees, + Av1ChromaPredictionMode.Smooth => Av1PredictionMode.Smooth, + Av1ChromaPredictionMode.SmoothVertical => Av1PredictionMode.SmoothVertical, + Av1ChromaPredictionMode.SmoothHorizontal => Av1PredictionMode.SmoothHorizontal, + Av1ChromaPredictionMode.Paeth => Av1PredictionMode.Paeth, + + // Chroma-from-luma adds its AC contribution to a DC prediction. libaom's get_uv_mode() therefore maps it + // to DC when shared transform and neighbor metadata require the corresponding luma predictor. + Av1ChromaPredictionMode.ChromaFromLuma => Av1PredictionMode.DC, + _ => Av1PredictionMode.IntraInvalid, + }; + + /// + /// Determines whether a chroma intra-prediction mode projects samples along a coded angle. + /// + /// The chroma intra-prediction mode. + /// for a directional mode; otherwise, . + public static bool IsDirectional(this Av1ChromaPredictionMode mode) + => mode is >= Av1ChromaPredictionMode.Vertical and <= Av1ChromaPredictionMode.Directional67Degrees; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs index 0b5098a38..f3ea62a65 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs @@ -13,6 +13,22 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; /// internal abstract partial class Av1FilterIntraPredictorBase { + /// + /// Defines the coefficient set for one AV1 filter-intra prediction mode. + /// + internal interface IAv1FilterIntraPredictionOperator + { + /// + /// Gets the filter-intra mode implemented by the operator. + /// + public static abstract Av1FilterIntraMode Mode { get; } + + /// + /// Gets the eight seven-tap coefficient rows used by the operator. + /// + public static abstract ReadOnlySpan Taps { get; } + } + /// /// Applies one closed filter-intra coefficient operator using the widest useful SIMD width. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.OperatorContract.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.OperatorContract.cs deleted file mode 100644 index 5e698f85b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.OperatorContract.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -internal abstract partial class Av1FilterIntraPredictorBase -{ - /// - /// Defines the coefficient set for one AV1 filter-intra prediction mode. - /// - internal interface IAv1FilterIntraPredictionOperator - { - /// - /// Gets the filter-intra mode implemented by the operator. - /// - public static abstract Av1FilterIntraMode Mode { get; } - - /// - /// Gets the eight seven-tap coefficient rows used by the operator. - /// - public static abstract ReadOnlySpan Taps { get; } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs index 5cb5640d3..7c6cb0c20 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs @@ -8,10 +8,182 @@ using System.Runtime.Intrinsics; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; /// -/// Provides the width-progressive SIMD traversal shared by closed non-directional prediction operators. +/// Defines the neighbor-usage flags and scalar/SIMD contract for closed intra-prediction operators, and provides +/// their shared width-progressive SIMD traversal. /// internal abstract partial class Av1IntraPredictorBase { + /// + /// Identifies the neighboring inputs consumed by an AV1 intra-prediction operator. + /// + [Flags] + internal enum Av1IntraPredictionInputs + { + /// + /// The operator does not consume neighboring samples. + /// + None = 0, + + /// + /// The operator consumes samples from the top reference. + /// + Top = 1, + + /// + /// The operator consumes samples from the left reference. + /// + Left = 2, + + /// + /// The operator consumes the shared top-left reference. + /// + TopLeft = 4, + + /// + /// The operator consumes the final top reference. + /// + TopRight = 8, + + /// + /// The operator consumes the final left reference. + /// + BottomLeft = 16, + + /// + /// The operator consumes the horizontal smooth weights. + /// + ColumnWeight = 32, + + /// + /// The operator consumes the vertical smooth weights. + /// + RowWeight = 64, + } + + /// + /// Defines the scalar and SIMD arithmetic for one non-directional AV1 intra-prediction mode. + /// + /// + /// Each overload performs the same lane-wise operation. The generic predictor traversal selects the widest + /// available overload, and the JIT specializes each static interface call for the closed operator type. + /// + internal interface IAv1IntraPredictionOperator + { + /// + /// Gets the prediction mode implemented by the operator. + /// + public static abstract Av1PredictionMode Mode { get; } + + /// + /// Gets the neighboring inputs consumed by the operator. + /// + public static abstract Av1IntraPredictionInputs Inputs { get; } + + /// + /// Predicts one 8-bit sample when hardware vectorization is unavailable. + /// + /// The top reference sample. + /// The left reference sample. + /// The shared top-left reference sample. + /// The final top reference sample. + /// The final left reference sample. + /// The horizontal Q8 smooth weight. + /// The vertical Q8 smooth weight. + /// The predicted sample. + public static abstract byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight); + + /// + /// Predicts sixteen 8-bit samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts thirty-two 8-bit samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts sixty-four 8-bit samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts one high-bit-depth sample when hardware vectorization is unavailable. + /// + /// The top reference sample. + /// The left reference sample. + /// The shared top-left reference sample. + /// The final top reference sample. + /// The final left reference sample. + /// The horizontal Q8 smooth weight. + /// The vertical Q8 smooth weight. + /// The predicted sample. + public static abstract short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight); + + /// + /// Predicts eight high-bit-depth samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts sixteen high-bit-depth samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight); + + /// + /// Predicts thirty-two high-bit-depth samples in parallel. + /// + /// The top reference samples. + /// The left reference sample in every lane. + /// The shared top-left reference sample in every lane. + /// The final top reference sample in every lane. + /// The final left reference sample in every lane. + /// The first horizontal Q8 smooth weight for these lanes. + /// The vertical Q8 smooth weight. + /// The predicted samples. + public static abstract Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight); + } + /// /// Applies one closed non-directional AV1 prediction operator using the widest available SIMD width. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs deleted file mode 100644 index 1533747b9..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; - -/// -/// Defines the neighbor-usage flags and scalar/SIMD contract for closed intra-prediction operators. -/// -internal abstract partial class Av1IntraPredictorBase -{ - /// - /// Identifies the neighboring inputs consumed by an AV1 intra-prediction operator. - /// - [Flags] - internal enum Av1IntraPredictionInputs - { - /// - /// The operator does not consume neighboring samples. - /// - None = 0, - - /// - /// The operator consumes samples from the top reference. - /// - Top = 1, - - /// - /// The operator consumes samples from the left reference. - /// - Left = 2, - - /// - /// The operator consumes the shared top-left reference. - /// - TopLeft = 4, - - /// - /// The operator consumes the final top reference. - /// - TopRight = 8, - - /// - /// The operator consumes the final left reference. - /// - BottomLeft = 16, - - /// - /// The operator consumes the horizontal smooth weights. - /// - ColumnWeight = 32, - - /// - /// The operator consumes the vertical smooth weights. - /// - RowWeight = 64, - } - - /// - /// Defines the scalar and SIMD arithmetic for one non-directional AV1 intra-prediction mode. - /// - /// - /// Each overload performs the same lane-wise operation. The generic predictor traversal selects the widest - /// available overload, and the JIT specializes each static interface call for the closed operator type. - /// - internal interface IAv1IntraPredictionOperator - { - /// - /// Gets the prediction mode implemented by the operator. - /// - public static abstract Av1PredictionMode Mode { get; } - - /// - /// Gets the neighboring inputs consumed by the operator. - /// - public static abstract Av1IntraPredictionInputs Inputs { get; } - - /// - /// Predicts one 8-bit sample when hardware vectorization is unavailable. - /// - /// The top reference sample. - /// The left reference sample. - /// The shared top-left reference sample. - /// The final top reference sample. - /// The final left reference sample. - /// The horizontal Q8 smooth weight. - /// The vertical Q8 smooth weight. - /// The predicted sample. - public static abstract byte Predict(byte top, byte left, byte topLeft, byte topRight, byte bottomLeft, int columnWeight, int rowWeight); - - /// - /// Predicts sixteen 8-bit samples in parallel. - /// - /// The top reference samples. - /// The left reference sample in every lane. - /// The shared top-left reference sample in every lane. - /// The final top reference sample in every lane. - /// The final left reference sample in every lane. - /// The first horizontal Q8 smooth weight for these lanes. - /// The vertical Q8 smooth weight. - /// The predicted samples. - public static abstract Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight); - - /// - /// Predicts thirty-two 8-bit samples in parallel. - /// - /// The top reference samples. - /// The left reference sample in every lane. - /// The shared top-left reference sample in every lane. - /// The final top reference sample in every lane. - /// The final left reference sample in every lane. - /// The first horizontal Q8 smooth weight for these lanes. - /// The vertical Q8 smooth weight. - /// The predicted samples. - public static abstract Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight); - - /// - /// Predicts sixty-four 8-bit samples in parallel. - /// - /// The top reference samples. - /// The left reference sample in every lane. - /// The shared top-left reference sample in every lane. - /// The final top reference sample in every lane. - /// The final left reference sample in every lane. - /// The first horizontal Q8 smooth weight for these lanes. - /// The vertical Q8 smooth weight. - /// The predicted samples. - public static abstract Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight); - - /// - /// Predicts one high-bit-depth sample when hardware vectorization is unavailable. - /// - /// The top reference sample. - /// The left reference sample. - /// The shared top-left reference sample. - /// The final top reference sample. - /// The final left reference sample. - /// The horizontal Q8 smooth weight. - /// The vertical Q8 smooth weight. - /// The predicted sample. - public static abstract short Predict(short top, short left, short topLeft, short topRight, short bottomLeft, int columnWeight, int rowWeight); - - /// - /// Predicts eight high-bit-depth samples in parallel. - /// - /// The top reference samples. - /// The left reference sample in every lane. - /// The shared top-left reference sample in every lane. - /// The final top reference sample in every lane. - /// The final left reference sample in every lane. - /// The first horizontal Q8 smooth weight for these lanes. - /// The vertical Q8 smooth weight. - /// The predicted samples. - public static abstract Vector128 Predict(Vector128 top, Vector128 left, Vector128 topLeft, Vector128 topRight, Vector128 bottomLeft, ref int columnWeights, int rowWeight); - - /// - /// Predicts sixteen high-bit-depth samples in parallel. - /// - /// The top reference samples. - /// The left reference sample in every lane. - /// The shared top-left reference sample in every lane. - /// The final top reference sample in every lane. - /// The final left reference sample in every lane. - /// The first horizontal Q8 smooth weight for these lanes. - /// The vertical Q8 smooth weight. - /// The predicted samples. - public static abstract Vector256 Predict(Vector256 top, Vector256 left, Vector256 topLeft, Vector256 topRight, Vector256 bottomLeft, ref int columnWeights, int rowWeight); - - /// - /// Predicts thirty-two high-bit-depth samples in parallel. - /// - /// The top reference samples. - /// The left reference sample in every lane. - /// The shared top-left reference sample in every lane. - /// The final top reference sample in every lane. - /// The final left reference sample in every lane. - /// The first horizontal Q8 smooth weight for these lanes. - /// The vertical Q8 smooth weight. - /// The predicted samples. - public static abstract Vector512 Predict(Vector512 top, Vector512 left, Vector512 topLeft, Vector512 topRight, Vector512 bottomLeft, ref int columnWeights, int rowWeight); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs index f9ffd1211..fea22f27d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs @@ -86,7 +86,7 @@ internal class Av1PredictionDecoder /// The transform block's vertical offset within the mode-information block. /// Corresponds to svt_av1_predict_intra in SVT-AV1. public void Decode( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1Plane plane, Av1TransformSize transformSize, Av1TileInfo tileInfo, @@ -96,7 +96,7 @@ internal class Av1PredictionDecoder int blockModeInfoColumnOffset, int blockModeInfoRowOffset) => this.DecodeCore( - partitionInfo, + ref partitionInfo, plane, transformSize, tileInfo, @@ -120,7 +120,7 @@ internal class Av1PredictionDecoder /// The transform block's vertical offset within the mode-information block. /// Implements the intra prediction portion of section 7.11.2 of the AV1 specification. public void Decode( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1Plane plane, Av1TransformSize transformSize, Av1TileInfo tileInfo, @@ -130,7 +130,7 @@ internal class Av1PredictionDecoder int blockModeInfoColumnOffset, int blockModeInfoRowOffset) => this.DecodeCore( - partitionInfo, + ref partitionInfo, plane, transformSize, tileInfo, @@ -154,7 +154,7 @@ internal class Av1PredictionDecoder /// The transform block's horizontal offset within the mode-information block. /// The transform block's vertical offset within the mode-information block. private void DecodeCore( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1Plane plane, Av1TransformSize transformSize, Av1TileInfo tileInfo, @@ -173,12 +173,11 @@ internal class Av1PredictionDecoder Span leftNeighbor = pixelBuffer[(stride - 1)..]; Span startOfPixels = pixelBuffer[stride..]; - Av1PredictionMode mode = (plane == Av1Plane.Y) ? partitionInfo.ModeInfo.YMode : partitionInfo.ModeInfo.UvMode; - - if (plane != Av1Plane.Y && partitionInfo.ModeInfo.UvMode == Av1PredictionMode.UvChromaFromLuma) + Av1PredictionMode mode = partitionInfo.ModeInfo.YMode; + if (plane != Av1Plane.Y && partitionInfo.ModeInfo.UvMode == Av1ChromaPredictionMode.ChromaFromLuma) { this.PredictIntraBlock( - partitionInfo, + ref partitionInfo, plane, transformSize, tileInfo, @@ -193,7 +192,7 @@ internal class Av1PredictionDecoder bitDepth); this.PredictChromaFromLumaBlock( - partitionInfo, + ref partitionInfo, partitionInfo.ChromaFromLumaContext, startOfPixels, stride, @@ -203,8 +202,15 @@ internal class Av1PredictionDecoder return; } + if (plane != Av1Plane.Y) + { + // Chroma and luma modes are separate bitstream domains. Shared spatial predictors consume the explicit + // libaom get_uv_mode() equivalent rather than relying on their matching ordinal values. + mode = partitionInfo.ModeInfo.UvMode.ToLumaMode(); + } + this.PredictIntraBlock( - partitionInfo, + ref partitionInfo, plane, transformSize, tileInfo, @@ -229,11 +235,17 @@ internal class Av1PredictionDecoder /// The distance, in samples, between pixel rows. /// The dimensions of the chroma transform block. /// The U or V plane being reconstructed. - private void PredictChromaFromLumaBlock(Av1PartitionInfo partitionInfo, Av1ChromaFromLumaContext? chromaFromLumaContext, Span pixelBuffer, int stride, Av1TransformSize transformSize, Av1Plane plane) + private void PredictChromaFromLumaBlock( + ref Av1PartitionInfo partitionInfo, + Av1ChromaFromLumaContext? chromaFromLumaContext, + Span pixelBuffer, + int stride, + Av1TransformSize transformSize, + Av1Plane plane) where T : unmanaged, IBinaryInteger { Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; - bool isChromaFromLumaAllowedFlag = IsChromaFromLumaAllowedWithFrameHeader(partitionInfo, this.sequenceHeader.ColorConfig, this.frameHeader); + bool isChromaFromLumaAllowedFlag = IsChromaFromLumaAllowedWithFrameHeader(ref partitionInfo, this.sequenceHeader.ColorConfig, this.frameHeader); DebugGuard.IsTrue(isChromaFromLumaAllowedFlag, "Chroma from Luma should be allowed then computing it."); if (chromaFromLumaContext == null) @@ -271,7 +283,7 @@ internal class Av1PredictionDecoder /// The sequence color configuration. /// The decoded frame header. /// when the block may use chroma-from-luma prediction; otherwise, . - private static bool IsChromaFromLumaAllowedWithFrameHeader(Av1PartitionInfo partitionInfo, ObuColorConfig colorConfig, ObuFrameHeader frameHeader) + private static bool IsChromaFromLumaAllowedWithFrameHeader(ref Av1PartitionInfo partitionInfo, ObuColorConfig colorConfig, ObuFrameHeader frameHeader) { Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; Av1BlockSize blockSize = modeInfo.BlockSize; @@ -327,7 +339,7 @@ internal class Av1PredictionDecoder /// The transform block's vertical offset within the mode-information block. /// The bit depth of the reconstructed samples. private void PredictIntraBlock( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1Plane plane, Av1TransformSize transformSize, Av1TileInfo tileInfo, @@ -359,7 +371,7 @@ internal class Av1PredictionDecoder { ReadOnlySpan paletteColors = modeInfo.GetPaletteColors(plane); ReadOnlySpan colorIndexMap = modeInfo.GetPaletteColorIndexMap(plane); - int paletteStride = partitionInfo.WidthInPixels[(int)plane]; + int paletteStride = partitionInfo.GetWidthInPixels(plane); int mapOffset = ((blockModeInfoRowOffset << Av1Constants.ModeInfoSizeLog2) * paletteStride) + (blockModeInfoColumnOffset << Av1Constants.ModeInfoSizeLog2); @@ -395,11 +407,13 @@ internal class Av1PredictionDecoder // These distances bound edge extension at the coded frame rather than allowing // a transform to read padding that happens to exist beyond the visible image. - int xr = (partitionInfo.ModeBlockToRightEdge >> (3 + subX)) + (partitionInfo.WidthInPixels[(int)plane] - (blockModeInfoColumnOffset << Av1Constants.ModeInfoSizeLog2) - transformWidth) - + int xr = (partitionInfo.ModeBlockToRightEdge >> (3 + subX)) + + (partitionInfo.GetWidthInPixels(plane) - (blockModeInfoColumnOffset << Av1Constants.ModeInfoSizeLog2) - transformWidth) - xrOffset; int yd = (partitionInfo.ModeBlockToBottomEdge >> (3 + subY)) + - (partitionInfo.HeightInPixels[(int)plane] - (blockModeInfoRowOffset << Av1Constants.ModeInfoSizeLog2) - transformHeight) - ydOffset; + (partitionInfo.GetHeightInPixels(plane) - (blockModeInfoRowOffset << Av1Constants.ModeInfoSizeLog2) - transformHeight) - ydOffset; + bool rightAvailable = modeInfoColumn + ((blockModeInfoColumnOffset + transformWidthInModeInfoUnits) << subX) < tileInfo.ModeInfoColumnEnd; bool bottomAvailable = (yd > 0) && (modeInfoRow + ((blockModeInfoRowOffset + transformHeightInModeInfoUnits) << subY) < tileInfo.ModeInfoRowEnd); @@ -439,7 +453,7 @@ internal class Av1PredictionDecoder // Calling all other intra predictors except CFL and palette. this.DecodeBuildIntraPredictors( - partitionInfo, + ref partitionInfo, topNeighbor, leftNeighbor, (nuint)referenceStride, @@ -752,7 +766,7 @@ internal class Av1PredictionDecoder /// The color plane being reconstructed. /// The number of bits used to represent each sample. private void DecodeBuildIntraPredictors( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Span aboveNeighbor, ReadOnlySpan leftNeighbor, nuint referenceStride, @@ -989,7 +1003,7 @@ internal class Av1PredictionDecoder bool needRight = angle < 90; bool needBottom = angle > 180; - bool filterType = GetFilterType(partitionInfo, plane); + bool filterType = GetFilterType(ref partitionInfo, plane); if (angle is not 90 and not 180) { @@ -1862,7 +1876,7 @@ internal class Av1PredictionDecoder /// The decoded partition and neighboring mode state. /// The color plane whose neighbors are inspected. /// when either relevant neighbor uses a smooth mode; otherwise, . - private static bool GetFilterType(Av1PartitionInfo partitionInfo, Av1Plane plane) + private static bool GetFilterType(ref Av1PartitionInfo partitionInfo, Av1Plane plane) { Av1BlockModeInfo? above; Av1BlockModeInfo? left; @@ -1900,10 +1914,10 @@ internal class Av1PredictionDecoder else { // Inter mode not supported here. - Av1PredictionMode uvMode = modeInfo.UvMode; - return uvMode is Av1PredictionMode.Smooth or - Av1PredictionMode.SmoothVertical or - Av1PredictionMode.SmoothHorizontal; + Av1ChromaPredictionMode uvMode = modeInfo.UvMode; + return uvMode is Av1ChromaPredictionMode.Smooth or + Av1ChromaPredictionMode.SmoothVertical or + Av1ChromaPredictionMode.SmoothHorizontal; } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs index d9e0a4805..720b78c79 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionMode.cs @@ -4,10 +4,9 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; /// -/// Identifies the intra-prediction modes used by an AV1 still-picture frame. +/// Identifies the luma intra and inter prediction modes used by an AV1 coding block. /// -/// Inter modes are omitted because reduced still-picture frames do not reference other frames. -internal enum Av1PredictionMode +internal enum Av1PredictionMode : byte { /// /// Predicts each sample from the average of the available top and left neighbors. @@ -75,9 +74,64 @@ internal enum Av1PredictionMode Paeth, /// - /// Predicts chroma from the reconstructed luma AC surface. + /// Uses the nearest motion-vector candidate for one reference frame. /// - UvChromaFromLuma, + NearestMotionVector = 13, + + /// + /// Uses a near motion-vector candidate for one reference frame. + /// + NearMotionVector = 14, + + /// + /// Uses the global-motion model for one reference frame. + /// + GlobalMotionVector = 15, + + /// + /// Decodes a new motion vector for one reference frame. + /// + NewMotionVector = 16, + + /// + /// Uses the nearest motion-vector candidate for both compound references. + /// + NearestNearestMotionVector = 17, + + /// + /// Uses a near motion-vector candidate for both compound references. + /// + NearNearMotionVector = 18, + + /// + /// Uses the nearest candidate for the first compound reference and decodes a new vector for the second. + /// + NearestNewMotionVector = 19, + + /// + /// Decodes a new vector for the first compound reference and uses the nearest candidate for the second. + /// + NewNearestMotionVector = 20, + + /// + /// Uses a near candidate for the first compound reference and decodes a new vector for the second. + /// + NearNewMotionVector = 21, + + /// + /// Decodes a new vector for the first compound reference and uses a near candidate for the second. + /// + NewNearMotionVector = 22, + + /// + /// Uses the global-motion model for both compound references. + /// + GlobalGlobalMotionVector = 23, + + /// + /// Decodes a new motion vector for both compound references. + /// + NewNewMotionVector = 24, /// /// The first luma intra-prediction mode. @@ -95,12 +149,42 @@ internal enum Av1PredictionMode IntraModes = Paeth + 1, /// - /// The number of chroma intra-prediction modes, including chroma-from-luma. + /// The first single-reference inter-prediction mode. + /// + SingleInterModeStart = NearestMotionVector, + + /// + /// The exclusive upper bound of single-reference inter-prediction modes. + /// + SingleInterModeEnd = NearestNearestMotionVector, + + /// + /// The first compound-reference inter-prediction mode. + /// + CompoundInterModeStart = NearestNearestMotionVector, + + /// + /// The exclusive upper bound of compound-reference inter-prediction modes. + /// + CompoundInterModeEnd = NewNewMotionVector + 1, + + /// + /// The first inter-prediction mode. + /// + InterModeStart = NearestMotionVector, + + /// + /// The exclusive upper bound of all inter-prediction modes. + /// + InterModeEnd = NewNewMotionVector + 1, + + /// + /// The number of luma and inter prediction modes in the complete AV1 mode domain. /// - UvIntraModes = UvChromaFromLuma + 1, + PredictionModeCount = NewNewMotionVector + 1, /// /// The invalid intra-mode sentinel matching the complete AV1 prediction-mode domain. /// - IntraInvalid = 25, + IntraInvalid = PredictionModeCount, } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs index 1076323ad..29c82b4ae 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Dispatch.cs @@ -28,8 +28,7 @@ internal static partial class Av1InterPredictor Av1InterpolationFilter verticalFilter, int horizontalPhase, int verticalPhase, - Span scratch, - bool scalarOnly) + Span scratch) { switch (horizontalFilter) { @@ -45,8 +44,7 @@ internal static partial class Av1InterPredictor verticalFilter, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Smooth: @@ -61,8 +59,7 @@ internal static partial class Av1InterPredictor verticalFilter, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Sharp: @@ -77,8 +74,7 @@ internal static partial class Av1InterPredictor verticalFilter, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; default: @@ -93,8 +89,7 @@ internal static partial class Av1InterPredictor verticalFilter, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; } @@ -116,8 +111,7 @@ internal static partial class Av1InterPredictor int horizontalPhase, int verticalPhase, int bitDepth, - Span scratch, - bool scalarOnly) + Span scratch) { switch (horizontalFilter) { @@ -134,8 +128,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Smooth: @@ -151,8 +144,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Sharp: @@ -168,8 +160,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; default: @@ -185,17 +176,16 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; } } /// - /// Copies an 8-bit integer-position block using the widest vector that fits a complete row prefix. + /// Selects a closed 8-bit horizontal interpolation operator for explicit scalar execution. /// - private static void Copy( + private static void DispatchScalar( ReadOnlySpan source, int sourceStride, int sourceOrigin, @@ -203,14 +193,179 @@ internal static partial class Av1InterPredictor int destinationStride, int width, int height, - bool scalarOnly) + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span scratch) { - if (scalarOnly) + // The benchmark/test entry point closes the same production operators explicitly, but terminates in the + // scalar kernels without carrying a runtime mode flag through the SIMD-first decoder path. + switch (horizontalFilter) { - CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); - return; + case Av1InterpolationFilter.Regular: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch); + + break; + case Av1InterpolationFilter.Smooth: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch); + + break; + case Av1InterpolationFilter.Sharp: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch); + + break; + default: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + scratch); + + break; } + } + /// + /// Selects a closed high-bit-depth horizontal interpolation operator for explicit scalar execution. + /// + private static void DispatchScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch) + { + // Closing the production table operators here keeps scalar parity coverage on the same normative Q7 data. + switch (horizontalFilter) + { + case Av1InterpolationFilter.Regular: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + case Av1InterpolationFilter.Smooth: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + case Av1InterpolationFilter.Sharp: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + default: + DispatchVerticalScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + } + } + + /// + /// Copies an 8-bit integer-position block using the widest vector that fits a complete row prefix. + /// + private static void Copy( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height) + { ref byte sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); ref byte destinationBase = ref MemoryMarshal.GetReference(destination); @@ -302,15 +457,8 @@ internal static partial class Av1InterPredictor Span destination, int destinationStride, int width, - int height, - bool scalarOnly) + int height) { - if (scalarOnly) - { - CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); - return; - } - ref ushort sourceBase = ref Unsafe.Add(ref MemoryMarshal.GetReference(source), sourceOrigin); ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); @@ -409,73 +557,69 @@ internal static partial class Av1InterPredictor int sourceOffset, int tapStride, int firstRound, - int secondRound, - bool scalarOnly) + int secondRound) { - if (!scalarOnly) + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) { - if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) - { - FilterDirect( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - coefficients, - tapCount, - sourceOffset, - tapStride, - firstRound, - secondRound, - Vector512.Zero); - - return; - } + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + Vector512.Zero); - if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) - { - FilterDirect( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - coefficients, - tapCount, - sourceOffset, - tapStride, - firstRound, - secondRound, - Vector256.Zero); - - return; - } + return; + } - if (Vector128.IsHardwareAccelerated) - { - FilterDirect( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - coefficients, - tapCount, - sourceOffset, - tapStride, - firstRound, - secondRound, - Vector128.Zero); - - return; - } + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + Vector256.Zero); + + return; + } + + if (Vector128.IsHardwareAccelerated) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + Vector128.Zero); + + return; } FilterDirectScalar( @@ -511,76 +655,72 @@ internal static partial class Av1InterPredictor int tapStride, int firstRound, int secondRound, - int bitDepth, - bool scalarOnly) + int bitDepth) { - if (!scalarOnly) + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) { - if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) - { - FilterDirect( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - coefficients, - tapCount, - sourceOffset, - tapStride, - firstRound, - secondRound, - bitDepth, - Vector512.Zero); + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth, + Vector512.Zero); - return; - } + return; + } - if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) - { - FilterDirect( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - coefficients, - tapCount, - sourceOffset, - tapStride, - firstRound, - secondRound, - bitDepth, - Vector256.Zero); + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth, + Vector256.Zero); - return; - } + return; + } - if (Vector128.IsHardwareAccelerated) - { - FilterDirect( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - coefficients, - tapCount, - sourceOffset, - tapStride, - firstRound, - secondRound, - bitDepth, - Vector128.Zero); + if (Vector128.IsHardwareAccelerated) + { + FilterDirect( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients, + tapCount, + sourceOffset, + tapStride, + firstRound, + secondRound, + bitDepth, + Vector128.Zero); - return; - } + return; } FilterDirectScalar( @@ -618,82 +758,78 @@ internal static partial class Av1InterPredictor int verticalTapCount, int verticalSourceOffset, int bitDepth, - Span scratch, - bool scalarOnly) + Span scratch) { - if (!scalarOnly) + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) { - if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) - { - Filter2D( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - horizontalCoefficients, - horizontalTapCount, - horizontalSourceOffset, - verticalCoefficients, - verticalTapCount, - verticalSourceOffset, - bitDepth, - Round0Bits, - scratch, - Vector512.Zero); + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch, + Vector512.Zero); - return; - } + return; + } - if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) - { - Filter2D( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - horizontalCoefficients, - horizontalTapCount, - horizontalSourceOffset, - verticalCoefficients, - verticalTapCount, - verticalSourceOffset, - bitDepth, - Round0Bits, - scratch, - Vector256.Zero); + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch, + Vector256.Zero); - return; - } + return; + } - if (Vector128.IsHardwareAccelerated) - { - Filter2D( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - horizontalCoefficients, - horizontalTapCount, - horizontalSourceOffset, - verticalCoefficients, - verticalTapCount, - verticalSourceOffset, - bitDepth, - Round0Bits, - scratch, - Vector128.Zero); + if (Vector128.IsHardwareAccelerated) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + Round0Bits, + scratch, + Vector128.Zero); - return; - } + return; } Filter2DScalar( @@ -734,82 +870,78 @@ internal static partial class Av1InterPredictor int verticalSourceOffset, int bitDepth, int round0, - Span scratch, - bool scalarOnly) + Span scratch) { - if (!scalarOnly) + if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) { - if (Vector512.IsHardwareAccelerated && Vector.Count == Vector512.Count && width >= Vector512.Count) - { - Filter2D( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - horizontalCoefficients, - horizontalTapCount, - horizontalSourceOffset, - verticalCoefficients, - verticalTapCount, - verticalSourceOffset, - bitDepth, - round0, - scratch, - Vector512.Zero); + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch, + Vector512.Zero); - return; - } + return; + } - if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) - { - Filter2D( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - horizontalCoefficients, - horizontalTapCount, - horizontalSourceOffset, - verticalCoefficients, - verticalTapCount, - verticalSourceOffset, - bitDepth, - round0, - scratch, - Vector256.Zero); + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch, + Vector256.Zero); - return; - } + return; + } - if (Vector128.IsHardwareAccelerated) - { - Filter2D( - source, - sourceStride, - sourceOrigin, - destination, - destinationStride, - width, - height, - horizontalCoefficients, - horizontalTapCount, - horizontalSourceOffset, - verticalCoefficients, - verticalTapCount, - verticalSourceOffset, - bitDepth, - round0, - scratch, - Vector128.Zero); + if (Vector128.IsHardwareAccelerated) + { + Filter2D( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients, + horizontalTapCount, + horizontalSourceOffset, + verticalCoefficients, + verticalTapCount, + verticalSourceOffset, + bitDepth, + round0, + scratch, + Vector128.Zero); - return; - } + return; } Filter2DScalar( diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs index a8739799d..c5a8649da 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.Operator.cs @@ -41,8 +41,7 @@ internal static partial class Av1InterPredictor Av1InterpolationFilter verticalFilter, int horizontalPhase, int verticalPhase, - Span scratch, - bool scalarOnly) + Span scratch) where THorizontal : struct, IAv1InterPredictorOperator { switch (verticalFilter) @@ -58,8 +57,7 @@ internal static partial class Av1InterPredictor height, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Smooth: @@ -73,8 +71,7 @@ internal static partial class Av1InterPredictor height, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Sharp: @@ -88,8 +85,7 @@ internal static partial class Av1InterPredictor height, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; default: @@ -103,8 +99,7 @@ internal static partial class Av1InterPredictor height, horizontalPhase, verticalPhase, - scratch, - scalarOnly); + scratch); break; } @@ -126,8 +121,7 @@ internal static partial class Av1InterPredictor int horizontalPhase, int verticalPhase, int bitDepth, - Span scratch, - bool scalarOnly) + Span scratch) where THorizontal : struct, IAv1InterPredictorOperator { switch (verticalFilter) @@ -144,8 +138,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Smooth: @@ -160,8 +153,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; case Av1InterpolationFilter.Sharp: @@ -176,8 +168,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); break; default: @@ -192,8 +183,170 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - scalarOnly); + scratch); + + break; + } + } + + /// + /// Selects an 8-bit vertical interpolation operator for explicit scalar execution. + /// + /// The closed horizontal filter family selected from the production operator set. + private static void DispatchVerticalScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span scratch) + where THorizontal : struct, IAv1InterPredictorOperator + { + switch (verticalFilter) + { + case Av1InterpolationFilter.Regular: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch); + + break; + case Av1InterpolationFilter.Smooth: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch); + + break; + case Av1InterpolationFilter.Sharp: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch); + + break; + default: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + scratch); + + break; + } + } + + /// + /// Selects a high-bit-depth vertical interpolation operator for explicit scalar execution. + /// + /// The closed horizontal filter family selected from the production operator set. + private static void DispatchVerticalScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch) + where THorizontal : struct, IAv1InterPredictorOperator + { + switch (verticalFilter) + { + case Av1InterpolationFilter.Regular: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + case Av1InterpolationFilter.Smooth: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + case Av1InterpolationFilter.Sharp: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); + + break; + default: + PredictScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalPhase, + verticalPhase, + bitDepth, + scratch); break; } @@ -214,14 +367,13 @@ internal static partial class Av1InterPredictor int height, int horizontalPhase, int verticalPhase, - Span scratch, - bool scalarOnly) + Span scratch) where THorizontal : struct, IAv1InterPredictorOperator where TVertical : struct, IAv1InterPredictorOperator { if (horizontalPhase == 0 && verticalPhase == 0) { - Copy(source, sourceStride, sourceOrigin, destination, destinationStride, width, height, scalarOnly); + Copy(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); return; } @@ -243,8 +395,7 @@ internal static partial class Av1InterPredictor firstCoefficient - 3, 1, Round0Bits, - FilterBits - Round0Bits, - scalarOnly); + FilterBits - Round0Bits); return; } @@ -267,8 +418,7 @@ internal static partial class Av1InterPredictor (firstCoefficient - 3) * sourceStride, sourceStride, FilterBits, - 0, - scalarOnly); + 0); return; } @@ -293,8 +443,7 @@ internal static partial class Av1InterPredictor verticalTapCount, firstVerticalCoefficient - 3, 8, - scratch, - scalarOnly); + scratch); } /// @@ -313,14 +462,13 @@ internal static partial class Av1InterPredictor int horizontalPhase, int verticalPhase, int bitDepth, - Span scratch, - bool scalarOnly) + Span scratch) where THorizontal : struct, IAv1InterPredictorOperator where TVertical : struct, IAv1InterPredictorOperator { if (horizontalPhase == 0 && verticalPhase == 0) { - Copy(source, sourceStride, sourceOrigin, destination, destinationStride, width, height, scalarOnly); + Copy(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); return; } @@ -348,8 +496,7 @@ internal static partial class Av1InterPredictor 1, round0, FilterBits - round0, - bitDepth, - scalarOnly); + bitDepth); return; } @@ -373,8 +520,7 @@ internal static partial class Av1InterPredictor sourceStride, FilterBits, 0, - bitDepth, - scalarOnly); + bitDepth); return; } @@ -400,8 +546,205 @@ internal static partial class Av1InterPredictor firstVerticalCoefficient - 3, bitDepth, round0, - scratch, - scalarOnly); + scratch); + } + + /// + /// Executes one closed 8-bit interpolation-filter pair through the explicit scalar kernels. + /// + /// The horizontal filter family. + /// The vertical filter family. + private static void PredictScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + int horizontalPhase, + int verticalPhase, + Span scratch) + where THorizontal : struct, IAv1InterPredictorOperator + where TVertical : struct, IAv1InterPredictorOperator + { + if (horizontalPhase == 0 && verticalPhase == 0) + { + CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); + return; + } + + if (verticalPhase == 0) + { + ReadOnlySpan coefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirectScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + firstCoefficient - 3, + 1, + Round0Bits, + FilterBits - Round0Bits); + + return; + } + + if (horizontalPhase == 0) + { + ReadOnlySpan coefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirectScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + (firstCoefficient - 3) * sourceStride, + sourceStride, + FilterBits, + 0); + + return; + } + + ReadOnlySpan horizontalCoefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + ReadOnlySpan verticalCoefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(horizontalCoefficients, out int firstHorizontalCoefficient, out int horizontalTapCount); + GetEffectiveKernel(verticalCoefficients, out int firstVerticalCoefficient, out int verticalTapCount); + + Filter2DScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients[firstHorizontalCoefficient..], + horizontalTapCount, + firstHorizontalCoefficient - 3, + verticalCoefficients[firstVerticalCoefficient..], + verticalTapCount, + firstVerticalCoefficient - 3, + 8, + Round0Bits, + scratch); + } + + /// + /// Executes one closed high-bit-depth interpolation-filter pair through the explicit scalar kernels. + /// + /// The horizontal filter family. + /// The vertical filter family. + private static void PredictScalar( + ReadOnlySpan source, + int sourceStride, + int sourceOrigin, + Span destination, + int destinationStride, + int width, + int height, + int horizontalPhase, + int verticalPhase, + int bitDepth, + Span scratch) + where THorizontal : struct, IAv1InterPredictorOperator + where TVertical : struct, IAv1InterPredictorOperator + { + if (horizontalPhase == 0 && verticalPhase == 0) + { + CopyScalar(source, sourceStride, sourceOrigin, destination, destinationStride, width, height); + return; + } + + // Scalar parity uses the same first-pass range correction as the SIMD traversal so twelve-bit + // intermediates remain signed-16-bit without changing the complete Q14 rounding distance. + int intermediateRange = bitDepth + FilterBits - Round0Bits + 2; + int round0 = Round0Bits + Math.Max(intermediateRange - 16, 0); + + if (verticalPhase == 0) + { + ReadOnlySpan coefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirectScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + firstCoefficient - 3, + 1, + round0, + FilterBits - round0, + bitDepth); + + return; + } + + if (horizontalPhase == 0) + { + ReadOnlySpan coefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(coefficients, out int firstCoefficient, out int tapCount); + + FilterDirectScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + coefficients[firstCoefficient..], + tapCount, + (firstCoefficient - 3) * sourceStride, + sourceStride, + FilterBits, + 0, + bitDepth); + + return; + } + + ReadOnlySpan horizontalCoefficients = THorizontal.GetCoefficients(horizontalPhase, width <= 4); + ReadOnlySpan verticalCoefficients = TVertical.GetCoefficients(verticalPhase, height <= 4); + GetEffectiveKernel(horizontalCoefficients, out int firstHorizontalCoefficient, out int horizontalTapCount); + GetEffectiveKernel(verticalCoefficients, out int firstVerticalCoefficient, out int verticalTapCount); + + Filter2DScalar( + source, + sourceStride, + sourceOrigin, + destination, + destinationStride, + width, + height, + horizontalCoefficients[firstHorizontalCoefficient..], + horizontalTapCount, + firstHorizontalCoefficient - 3, + verticalCoefficients[firstVerticalCoefficient..], + verticalTapCount, + firstVerticalCoefficient - 3, + bitDepth, + round0, + scratch); } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs index 8c43adbec..a5af87360 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterPredictor.cs @@ -92,8 +92,7 @@ internal static partial class Av1InterPredictor verticalFilter, horizontalPhase, verticalPhase, - scratch, - false); + scratch); /// /// Reconstructs an 8-, 10-, or 12-bit translational prediction using the widest supported SIMD kernel. @@ -140,8 +139,7 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - false); + scratch); /// /// Reconstructs an 8-bit translational prediction without explicit hardware intrinsics. @@ -173,7 +171,7 @@ internal static partial class Av1InterPredictor int horizontalPhase, int verticalPhase, Span scratch) - => Dispatch( + => DispatchScalar( source, sourceStride, sourceOrigin, @@ -185,8 +183,7 @@ internal static partial class Av1InterPredictor verticalFilter, horizontalPhase, verticalPhase, - scratch, - true); + scratch); /// /// Reconstructs an 8-, 10-, or 12-bit translational prediction without explicit hardware intrinsics. @@ -220,7 +217,7 @@ internal static partial class Av1InterPredictor int verticalPhase, int bitDepth, Span scratch) - => Dispatch( + => DispatchScalar( source, sourceStride, sourceOrigin, @@ -233,6 +230,5 @@ internal static partial class Av1InterPredictor horizontalPhase, verticalPhase, bitDepth, - scratch, - true); + scratch); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs index b2bc9c5fa..963d4dde9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1InterpolationFilter.cs @@ -4,27 +4,32 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; /// -/// Identifies an AV1 interpolation filter used for translational inter prediction. +/// Identifies an AV1 interpolation-filter family or the frame-level switchable selection. /// -internal enum Av1InterpolationFilter +internal enum Av1InterpolationFilter : byte { /// /// The regular interpolation-filter family. /// - Regular, + Regular = 0, /// /// The smooth interpolation-filter family. /// - Smooth, + Smooth = 1, /// /// The sharp interpolation-filter family. /// - Sharp, + Sharp = 2, /// /// The bilinear interpolation-filter family. /// - Bilinear, + Bilinear = 3, + + /// + /// Indicates that each inter block selects its interpolation-filter family. + /// + Switchable = 4, } diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operators.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.BilinearOperator.cs similarity index 62% rename from src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operators.cs rename to src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.BilinearOperator.cs index 42f2bd993..2e2fa0d78 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operators.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.BilinearOperator.cs @@ -5,143 +5,8 @@ using System.Runtime.Intrinsics; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; -/// -/// Defines the closed interpolation operators used by intra-block-copy prediction. -/// internal static partial class Av1IntraBlockCopyPredictor { - /// - /// Averages horizontally adjacent source samples for a half-sample horizontal phase. - /// - private readonly struct HorizontalOperator : IOperator - { - /// - public static bool UsesRight => true; - - /// - public static bool UsesBottom => false; - - /// - public static byte Filter(byte topLeft, byte topRight, byte bottomLeft, byte bottomRight) => (byte)((topLeft + topRight + 1) >> 1); - - /// - public static Vector128 Filter( - Vector128 topLeft, - Vector128 topRight, - Vector128 bottomLeft, - Vector128 bottomRight) - => AverageRounded(topLeft, topRight); - - /// - public static Vector256 Filter( - Vector256 topLeft, - Vector256 topRight, - Vector256 bottomLeft, - Vector256 bottomRight) - => AverageRounded(topLeft, topRight); - - /// - public static Vector512 Filter( - Vector512 topLeft, - Vector512 topRight, - Vector512 bottomLeft, - Vector512 bottomRight) - => AverageRounded(topLeft, topRight); - - /// - public static short Filter(short topLeft, short topRight, short bottomLeft, short bottomRight) => (short)((topLeft + topRight + 1) >> 1); - - /// - public static Vector128 Filter( - Vector128 topLeft, - Vector128 topRight, - Vector128 bottomLeft, - Vector128 bottomRight) - => AverageRounded(topLeft, topRight); - - /// - public static Vector256 Filter( - Vector256 topLeft, - Vector256 topRight, - Vector256 bottomLeft, - Vector256 bottomRight) - => AverageRounded(topLeft, topRight); - - /// - public static Vector512 Filter( - Vector512 topLeft, - Vector512 topRight, - Vector512 bottomLeft, - Vector512 bottomRight) - => AverageRounded(topLeft, topRight); - } - - /// - /// Averages vertically adjacent source samples for a half-sample vertical phase. - /// - private readonly struct VerticalOperator : IOperator - { - /// - public static bool UsesRight => false; - - /// - public static bool UsesBottom => true; - - /// - public static byte Filter(byte topLeft, byte topRight, byte bottomLeft, byte bottomRight) => (byte)((topLeft + bottomLeft + 1) >> 1); - - /// - public static Vector128 Filter( - Vector128 topLeft, - Vector128 topRight, - Vector128 bottomLeft, - Vector128 bottomRight) - => AverageRounded(topLeft, bottomLeft); - - /// - public static Vector256 Filter( - Vector256 topLeft, - Vector256 topRight, - Vector256 bottomLeft, - Vector256 bottomRight) - => AverageRounded(topLeft, bottomLeft); - - /// - public static Vector512 Filter( - Vector512 topLeft, - Vector512 topRight, - Vector512 bottomLeft, - Vector512 bottomRight) - => AverageRounded(topLeft, bottomLeft); - - /// - public static short Filter(short topLeft, short topRight, short bottomLeft, short bottomRight) => (short)((topLeft + bottomLeft + 1) >> 1); - - /// - public static Vector128 Filter( - Vector128 topLeft, - Vector128 topRight, - Vector128 bottomLeft, - Vector128 bottomRight) - => AverageRounded(topLeft, bottomLeft); - - /// - public static Vector256 Filter( - Vector256 topLeft, - Vector256 topRight, - Vector256 bottomLeft, - Vector256 bottomRight) - => AverageRounded(topLeft, bottomLeft); - - /// - public static Vector512 Filter( - Vector512 topLeft, - Vector512 topRight, - Vector512 bottomLeft, - Vector512 bottomRight) - => AverageRounded(topLeft, bottomLeft); - } - /// /// Applies the separable two-dimensional interpolation required when both source axes have a half-sample phase. /// @@ -154,7 +19,7 @@ internal static partial class Av1IntraBlockCopyPredictor /// unsigned 32-bit halves. Narrowing recombines those halves in source-column order after the rounded result has /// returned to the original sample range. /// - private readonly struct BilinearOperator : IOperator + private readonly struct BilinearOperator : IAv1IntraBlockCopyOperator { /// public static bool UsesRight => true; diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.HorizontalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.HorizontalOperator.cs new file mode 100644 index 000000000..ae9d2994c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.HorizontalOperator.cs @@ -0,0 +1,78 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; + +/// +/// Defines the closed interpolation operators used by intra-block-copy prediction. +/// +internal static partial class Av1IntraBlockCopyPredictor +{ + /// + /// Averages horizontally adjacent source samples for a half-sample horizontal phase. + /// + private readonly struct HorizontalOperator : IAv1IntraBlockCopyOperator + { + /// + public static bool UsesRight => true; + + /// + public static bool UsesBottom => false; + + /// + public static byte Filter(byte topLeft, byte topRight, byte bottomLeft, byte bottomRight) => (byte)((topLeft + topRight + 1) >> 1); + + /// + public static Vector128 Filter( + Vector128 topLeft, + Vector128 topRight, + Vector128 bottomLeft, + Vector128 bottomRight) + => AverageRounded(topLeft, topRight); + + /// + public static Vector256 Filter( + Vector256 topLeft, + Vector256 topRight, + Vector256 bottomLeft, + Vector256 bottomRight) + => AverageRounded(topLeft, topRight); + + /// + public static Vector512 Filter( + Vector512 topLeft, + Vector512 topRight, + Vector512 bottomLeft, + Vector512 bottomRight) + => AverageRounded(topLeft, topRight); + + /// + public static short Filter(short topLeft, short topRight, short bottomLeft, short bottomRight) => (short)((topLeft + topRight + 1) >> 1); + + /// + public static Vector128 Filter( + Vector128 topLeft, + Vector128 topRight, + Vector128 bottomLeft, + Vector128 bottomRight) + => AverageRounded(topLeft, topRight); + + /// + public static Vector256 Filter( + Vector256 topLeft, + Vector256 topRight, + Vector256 bottomLeft, + Vector256 bottomRight) + => AverageRounded(topLeft, topRight); + + /// + public static Vector512 Filter( + Vector512 topLeft, + Vector512 topRight, + Vector512 bottomLeft, + Vector512 bottomRight) + => AverageRounded(topLeft, topRight); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operator.cs index 5dddbe2ec..013cedd14 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.Operator.cs @@ -8,10 +8,135 @@ using System.Runtime.Intrinsics; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; /// -/// Provides the width-progressive SIMD traversal shared by the intra-block-copy interpolation operators. +/// Defines the scalar and SIMD contract for closed intra-block-copy filter operators, and provides their shared +/// width-progressive SIMD traversal. /// internal static partial class Av1IntraBlockCopyPredictor { + /// + /// Defines lane-wise arithmetic for one intra-block-copy filter phase. + /// + /// + /// Every SIMD lane corresponds to one output column. The generic traversal supplies the integer source sample and + /// its right, lower, and lower-right neighbors; closed operator types allow the JIT to remove unused source loads. + /// + private interface IAv1IntraBlockCopyOperator + { + /// + /// Gets a value indicating whether the operator consumes the source sample to the right. + /// + public static abstract bool UsesRight { get; } + + /// + /// Gets a value indicating whether the operator consumes the source sample on the following row. + /// + public static abstract bool UsesBottom { get; } + + /// + /// Filters one 8-bit sample. + /// + /// The integer-position source sample. + /// The source sample one column to the right. + /// The source sample one row below. + /// The source sample one row below and one column to the right. + /// The filtered 8-bit sample. + public static abstract byte Filter(byte topLeft, byte topRight, byte bottomLeft, byte bottomRight); + + /// + /// Filters sixteen 8-bit samples in parallel. + /// + /// The integer-position source samples. + /// The source samples one column to the right. + /// The source samples one row below. + /// The source samples one row below and one column to the right. + /// The filtered 8-bit samples. + public static abstract Vector128 Filter( + Vector128 topLeft, + Vector128 topRight, + Vector128 bottomLeft, + Vector128 bottomRight); + + /// + /// Filters thirty-two 8-bit samples in parallel. + /// + /// The integer-position source samples. + /// The source samples one column to the right. + /// The source samples one row below. + /// The source samples one row below and one column to the right. + /// The filtered 8-bit samples. + public static abstract Vector256 Filter( + Vector256 topLeft, + Vector256 topRight, + Vector256 bottomLeft, + Vector256 bottomRight); + + /// + /// Filters sixty-four 8-bit samples in parallel. + /// + /// The integer-position source samples. + /// The source samples one column to the right. + /// The source samples one row below. + /// The source samples one row below and one column to the right. + /// The filtered 8-bit samples. + public static abstract Vector512 Filter( + Vector512 topLeft, + Vector512 topRight, + Vector512 bottomLeft, + Vector512 bottomRight); + + /// + /// Filters one high-bit-depth sample. + /// + /// The integer-position source sample. + /// The source sample one column to the right. + /// The source sample one row below. + /// The source sample one row below and one column to the right. + /// The filtered high-bit-depth sample. + public static abstract short Filter(short topLeft, short topRight, short bottomLeft, short bottomRight); + + /// + /// Filters eight high-bit-depth samples in parallel. + /// + /// The integer-position source samples. + /// The source samples one column to the right. + /// The source samples one row below. + /// The source samples one row below and one column to the right. + /// The filtered high-bit-depth samples. + public static abstract Vector128 Filter( + Vector128 topLeft, + Vector128 topRight, + Vector128 bottomLeft, + Vector128 bottomRight); + + /// + /// Filters sixteen high-bit-depth samples in parallel. + /// + /// The integer-position source samples. + /// The source samples one column to the right. + /// The source samples one row below. + /// The source samples one row below and one column to the right. + /// The filtered high-bit-depth samples. + public static abstract Vector256 Filter( + Vector256 topLeft, + Vector256 topRight, + Vector256 bottomLeft, + Vector256 bottomRight); + + /// + /// Filters thirty-two high-bit-depth samples in parallel. + /// + /// The integer-position source samples. + /// The source samples one column to the right. + /// The source samples one row below. + /// The source samples one row below and one column to the right. + /// The filtered high-bit-depth samples. + public static abstract Vector512 Filter( + Vector512 topLeft, + Vector512 topRight, + Vector512 bottomLeft, + Vector512 bottomRight); + } + /// /// Applies one closed interpolation operator to an 8-bit source block. /// @@ -23,7 +148,7 @@ internal static partial class Av1IntraBlockCopyPredictor int destinationStride, int width, int height) - where TOperator : struct, IOperator + where TOperator : struct, IAv1IntraBlockCopyOperator { ref byte sourceBase = ref MemoryMarshal.GetReference(source); ref byte destinationBase = ref MemoryMarshal.GetReference(destination); @@ -181,7 +306,7 @@ internal static partial class Av1IntraBlockCopyPredictor int destinationStride, int width, int height) - where TOperator : struct, IOperator + where TOperator : struct, IAv1IntraBlockCopyOperator { ref short sourceBase = ref MemoryMarshal.GetReference(source); ref short destinationBase = ref MemoryMarshal.GetReference(destination); @@ -329,7 +454,7 @@ internal static partial class Av1IntraBlockCopyPredictor int destinationStride, int width, int height) - where TOperator : struct, IOperator + where TOperator : struct, IAv1IntraBlockCopyOperator { for (int row = 0; row < height; row++) { @@ -358,7 +483,7 @@ internal static partial class Av1IntraBlockCopyPredictor int destinationStride, int width, int height) - where TOperator : struct, IOperator + where TOperator : struct, IAv1IntraBlockCopyOperator { for (int row = 0; row < height; row++) { diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.OperatorContract.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.OperatorContract.cs deleted file mode 100644 index 48aea9f3b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.OperatorContract.cs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; - -/// -/// Defines the scalar and SIMD contract for closed intra-block-copy filter operators. -/// -internal static partial class Av1IntraBlockCopyPredictor -{ - /// - /// Defines lane-wise arithmetic for one intra-block-copy filter phase. - /// - /// - /// Every SIMD lane corresponds to one output column. The generic traversal supplies the integer source sample and - /// its right, lower, and lower-right neighbors; closed operator types allow the JIT to remove unused source loads. - /// - private interface IOperator - { - /// - /// Gets a value indicating whether the operator consumes the source sample to the right. - /// - public static abstract bool UsesRight { get; } - - /// - /// Gets a value indicating whether the operator consumes the source sample on the following row. - /// - public static abstract bool UsesBottom { get; } - - /// - /// Filters one 8-bit sample. - /// - /// The integer-position source sample. - /// The source sample one column to the right. - /// The source sample one row below. - /// The source sample one row below and one column to the right. - /// The filtered 8-bit sample. - public static abstract byte Filter(byte topLeft, byte topRight, byte bottomLeft, byte bottomRight); - - /// - /// Filters sixteen 8-bit samples in parallel. - /// - /// The integer-position source samples. - /// The source samples one column to the right. - /// The source samples one row below. - /// The source samples one row below and one column to the right. - /// The filtered 8-bit samples. - public static abstract Vector128 Filter( - Vector128 topLeft, - Vector128 topRight, - Vector128 bottomLeft, - Vector128 bottomRight); - - /// - /// Filters thirty-two 8-bit samples in parallel. - /// - /// The integer-position source samples. - /// The source samples one column to the right. - /// The source samples one row below. - /// The source samples one row below and one column to the right. - /// The filtered 8-bit samples. - public static abstract Vector256 Filter( - Vector256 topLeft, - Vector256 topRight, - Vector256 bottomLeft, - Vector256 bottomRight); - - /// - /// Filters sixty-four 8-bit samples in parallel. - /// - /// The integer-position source samples. - /// The source samples one column to the right. - /// The source samples one row below. - /// The source samples one row below and one column to the right. - /// The filtered 8-bit samples. - public static abstract Vector512 Filter( - Vector512 topLeft, - Vector512 topRight, - Vector512 bottomLeft, - Vector512 bottomRight); - - /// - /// Filters one high-bit-depth sample. - /// - /// The integer-position source sample. - /// The source sample one column to the right. - /// The source sample one row below. - /// The source sample one row below and one column to the right. - /// The filtered high-bit-depth sample. - public static abstract short Filter(short topLeft, short topRight, short bottomLeft, short bottomRight); - - /// - /// Filters eight high-bit-depth samples in parallel. - /// - /// The integer-position source samples. - /// The source samples one column to the right. - /// The source samples one row below. - /// The source samples one row below and one column to the right. - /// The filtered high-bit-depth samples. - public static abstract Vector128 Filter( - Vector128 topLeft, - Vector128 topRight, - Vector128 bottomLeft, - Vector128 bottomRight); - - /// - /// Filters sixteen high-bit-depth samples in parallel. - /// - /// The integer-position source samples. - /// The source samples one column to the right. - /// The source samples one row below. - /// The source samples one row below and one column to the right. - /// The filtered high-bit-depth samples. - public static abstract Vector256 Filter( - Vector256 topLeft, - Vector256 topRight, - Vector256 bottomLeft, - Vector256 bottomRight); - - /// - /// Filters thirty-two high-bit-depth samples in parallel. - /// - /// The integer-position source samples. - /// The source samples one column to the right. - /// The source samples one row below. - /// The source samples one row below and one column to the right. - /// The filtered high-bit-depth samples. - public static abstract Vector512 Filter( - Vector512 topLeft, - Vector512 topRight, - Vector512 bottomLeft, - Vector512 bottomRight); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.VerticalOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.VerticalOperator.cs new file mode 100644 index 000000000..27088106a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/IntraBlockCopy/Av1IntraBlockCopyPredictor.VerticalOperator.cs @@ -0,0 +1,75 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; + +internal static partial class Av1IntraBlockCopyPredictor +{ + /// + /// Averages vertically adjacent source samples for a half-sample vertical phase. + /// + private readonly struct VerticalOperator : IAv1IntraBlockCopyOperator + { + /// + public static bool UsesRight => false; + + /// + public static bool UsesBottom => true; + + /// + public static byte Filter(byte topLeft, byte topRight, byte bottomLeft, byte bottomRight) => (byte)((topLeft + bottomLeft + 1) >> 1); + + /// + public static Vector128 Filter( + Vector128 topLeft, + Vector128 topRight, + Vector128 bottomLeft, + Vector128 bottomRight) + => AverageRounded(topLeft, bottomLeft); + + /// + public static Vector256 Filter( + Vector256 topLeft, + Vector256 topRight, + Vector256 bottomLeft, + Vector256 bottomRight) + => AverageRounded(topLeft, bottomLeft); + + /// + public static Vector512 Filter( + Vector512 topLeft, + Vector512 topRight, + Vector512 bottomLeft, + Vector512 bottomRight) + => AverageRounded(topLeft, bottomLeft); + + /// + public static short Filter(short topLeft, short topRight, short bottomLeft, short bottomRight) => (short)((topLeft + bottomLeft + 1) >> 1); + + /// + public static Vector128 Filter( + Vector128 topLeft, + Vector128 topRight, + Vector128 bottomLeft, + Vector128 bottomRight) + => AverageRounded(topLeft, bottomLeft); + + /// + public static Vector256 Filter( + Vector256 topLeft, + Vector256 topRight, + Vector256 bottomLeft, + Vector256 bottomRight) + => AverageRounded(topLeft, bottomLeft); + + /// + public static Vector512 Filter( + Vector512 topLeft, + Vector512 topRight, + Vector512 bottomLeft, + Vector512 bottomRight) + => AverageRounded(topLeft, bottomLeft); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrame.cs b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrame.cs new file mode 100644 index 000000000..a1953c483 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrame.cs @@ -0,0 +1,135 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; + +/// +/// Owns the completed decoded state retained for one AV1 reference or presentation frame. +/// +/// +/// Reference-map owners contain reconstruction samples after the normative in-loop filters and before film-grain +/// synthesis. A presentation-only owner may instead contain the independently synthesized grained output. +/// +internal sealed class Av1ReferenceFrame : IDisposable +{ + /// + /// The completed sample planes while this instance owns them. + /// + private Av1FrameBuffer? frameBuffer; + + /// + /// The independently retained entropy snapshot while this frame owner remains alive. + /// + private Av1FrameEntropyContext? entropyContext; + + /// + /// The decoder-session owner that receives when this frame is released. + /// + private Av1FrameEntropyContexts? entropyContextOwner; + + /// + /// Initializes a new instance of the class and takes ownership of the decoded + /// sample buffer. + /// + /// + /// The completed sample buffer. Ownership transfers to this instance when construction succeeds. + /// + /// + /// The completed frame header associated with the reconstructed samples. The caller must not mutate the header + /// after transferring it to this instance. + /// + /// + /// The completed per-block state associated with the reconstructed samples. The caller must not mutate the state + /// after transferring it to this instance. + /// + public Av1ReferenceFrame(Av1FrameBuffer frameBuffer, ObuFrameHeader frameHeader, Av1FrameInfo frameInfo) + { + this.frameBuffer = frameBuffer; + this.FrameHeader = frameHeader; + this.FrameInfo = frameInfo; + } + + /// + /// Initializes a new instance of the class and takes ownership of decoded samples + /// and the entropy snapshot retained by a refreshed reference frame. + /// + /// + /// The completed sample buffer. Ownership transfers to this instance when construction succeeds. + /// + /// + /// The completed frame header associated with the reconstructed samples. The caller must not mutate the header + /// after transferring it to this instance. + /// + /// + /// The completed per-block state associated with the reconstructed samples. The caller must not mutate the state + /// after transferring it to this instance. + /// + /// The completed entropy snapshot selected for later primary-reference use. + /// The decoder-session owner to which the snapshot is returned. + public Av1ReferenceFrame( + Av1FrameBuffer frameBuffer, + ObuFrameHeader frameHeader, + Av1FrameInfo frameInfo, + Av1FrameEntropyContext entropyContext, + Av1FrameEntropyContexts entropyContextOwner) + : this(frameBuffer, frameHeader, frameInfo) + { + this.entropyContext = entropyContext; + this.entropyContextOwner = entropyContextOwner; + } + + /// + /// Gets the completed sample buffer owned by this frame. + /// + public Av1FrameBuffer FrameBuffer => this.frameBuffer!; + + /// + /// Gets the completed header that describes the retained frame. + /// + public ObuFrameHeader FrameHeader { get; } + + /// + /// Gets the decoded per-block mode, motion, transform, and filter state associated with the retained frame. + /// + public Av1FrameInfo FrameInfo { get; } + + /// + /// Gets the entropy context retained for primary-reference use, or for a presentation-only + /// frame. + /// + public Av1FrameEntropyContext? EntropyContext => this.entropyContext; + + /// + /// Transfers the completed sample planes out of this frame owner. + /// + /// The completed sample planes now owned by the caller. + public Av1FrameBuffer TakeFrameBuffer() + { + Av1FrameBuffer result = this.frameBuffer!; + this.frameBuffer = null; + return result; + } + + /// + /// Releases the owned completed sample planes and returns any retained entropy snapshot to its decoder session. + /// + public void Dispose() + { + Av1FrameEntropyContext? context = this.entropyContext; + this.entropyContext = null; + if (context is not null) + { + // Nulling the field before returning the graph makes repeated disposal harmless and guarantees that one + // shared frame owner occupying multiple reference slots returns its snapshot exactly once. + this.entropyContextOwner!.ReturnSnapshot(context); + this.entropyContextOwner = null; + } + + this.frameBuffer?.Dispose(); + this.frameBuffer = null; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameBorder.cs b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameBorder.cs new file mode 100644 index 000000000..d3912f44b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameBorder.cs @@ -0,0 +1,111 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; + +/// +/// Extends reconstructed AV1 edge samples through the padded reference-frame border. +/// +internal static class Av1ReferenceFrameBorder +{ + /// + /// Replicates every visible plane edge through its complete decoder padding. + /// + /// The post-restoration reference frame whose padding is extended. + public static void Extend(Av1FrameBuffer frameBuffer) + { + ObuColorConfig colorConfig = frameBuffer.ColorConfig; + int subsamplingX = !colorConfig.IsMonochrome && colorConfig.SubSamplingX ? 1 : 0; + int subsamplingY = !colorConfig.IsMonochrome && colorConfig.SubSamplingY ? 1 : 0; + + ExtendPlane( + frameBuffer, + frameBuffer.BufferY!, + frameBuffer.OriginX, + frameBuffer.OriginY, + frameBuffer.Width, + frameBuffer.Height); + + if (!colorConfig.IsMonochrome) + { + int chromaWidth = Av1Math.DivideLog2Ceiling(frameBuffer.Width, subsamplingX); + int chromaHeight = Av1Math.DivideLog2Ceiling(frameBuffer.Height, subsamplingY); + int chromaOriginX = frameBuffer.OriginX >> subsamplingX; + int chromaOriginY = frameBuffer.OriginY >> subsamplingY; + + ExtendPlane(frameBuffer, frameBuffer.BufferCb!, chromaOriginX, chromaOriginY, chromaWidth, chromaHeight); + ExtendPlane(frameBuffer, frameBuffer.BufferCr!, chromaOriginX, chromaOriginY, chromaWidth, chromaHeight); + } + } + + /// + /// Selects the native sample representation for one byte-backed plane. + /// + /// The frame that defines the native sample size. + /// The padded plane allocation. + /// The horizontal visible origin in plane samples. + /// The vertical visible origin in rows. + /// The visible plane width. + /// The visible plane height. + private static void ExtendPlane( + Av1FrameBuffer frameBuffer, + Buffer2D buffer, + int originX, + int originY, + int width, + int height) + { + if (frameBuffer.BytesPerSample == 2) + { + ExtendPlane(MemoryMarshal.Cast(buffer.DangerousGetSingleSpan()), buffer.Width >> 1, originX, originY, width, height); + } + else + { + ExtendPlane(buffer.DangerousGetSingleSpan(), buffer.Width, originX, originY, width, height); + } + } + + /// + /// Extends one native sample plane horizontally and then vertically. + /// + /// The native eight-bit or high-bit-depth sample type. + /// The complete padded plane allocation. + /// The number of native samples between adjacent rows. + /// The horizontal visible origin in plane samples. + /// The vertical visible origin in rows. + /// The visible plane width. + /// The visible plane height. + private static void ExtendPlane(Span plane, int stride, int originX, int originY, int width, int height) + where TSample : unmanaged + { + int rightStart = originX + width; + int rightLength = stride - rightStart; + + for (int row = 0; row < height; row++) + { + Span destinationRow = plane.Slice((originY + row) * stride, stride); + + // Span.Fill maps these long constant runs to the runtime's vectorized fill implementation. Extending the + // horizontal edges first also makes each later full-row copy include complete left and right padding. + destinationRow[..originX].Fill(destinationRow[originX]); + destinationRow.Slice(rightStart, rightLength).Fill(destinationRow[rightStart - 1]); + } + + ReadOnlySpan firstVisibleRow = plane.Slice(originY * stride, stride); + for (int row = 0; row < originY; row++) + { + firstVisibleRow.CopyTo(plane.Slice(row * stride, stride)); + } + + int bottomStart = originY + height; + ReadOnlySpan lastVisibleRow = plane.Slice((bottomStart - 1) * stride, stride); + for (int row = bottomStart; row < plane.Length / stride; row++) + { + lastVisibleRow.CopyTo(plane.Slice(row * stride, stride)); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameDerivation.cs b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameDerivation.cs new file mode 100644 index 000000000..01149107e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameDerivation.cs @@ -0,0 +1,279 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; + +/// +/// Derives the seven AV1 inter-reference map indices from short reference signaling. +/// +internal static class Av1ReferenceFrameDerivation +{ + /// + /// The shifted-order sentinel used for a reference-map slot that is not available to the current frame. + /// + private const int UnavailableSortIndex = -1; + + /// + /// The reference-type value subtracted when indexing the seven-entry inter-reference map. + /// + private const int ReferenceIndexOffset = (int)Av1ReferenceFrameType.Last; + + /// + /// Gets the order in which unassigned backward roles are replaced by forward references. + /// + private static ReadOnlySpan RemainingReferenceOrder => + [ + Av1ReferenceFrameType.Last2, + Av1ReferenceFrameType.Last3, + Av1ReferenceFrameType.Backward, + Av1ReferenceFrameType.Alternate2, + Av1ReferenceFrameType.Alternate, + ]; + + /// + /// Derives the reference-map slot selected for each inter-reference type when an AV1 frame uses short reference + /// signaling. + /// + /// The current frame order hint in the active modulo order-hint domain. + /// The number of bits in the active order-hint domain. + /// The explicitly signaled reference-map slot for . + /// The explicitly signaled reference-map slot for . + /// The eight persisted reference-map order hints. + /// + /// The eight values indicating whether each persisted reference-map slot owns a decoded frame. An empty slot is + /// excluded from derivation. + /// + /// + /// The destination for seven slot indices ordered from through + /// . + /// + /// + /// The signaled LAST or GOLDEN slot is empty, refers to the current frame, or refers to a future frame. + /// + /// + /// The caller owns the fixed AV1 table-size invariants: and + /// contain eight entries, while contains + /// seven entries. The signaled indices and order hints have already been read from their bounded bit fields. Every + /// destination entry is overwritten on success, and multiple reference types may select the same slot. Frame-ID + /// validity is a separate conformance state that the caller checks for every resolved reference after derivation. + /// + public static void DeriveShortSignaledReferences( + uint currentOrderHint, + int orderHintBitWidth, + uint lastFrameIndex, + uint goldenFrameIndex, + ReadOnlySpan slotOrderHints, + ReadOnlySpan slotOccupancy, + Span referenceFrameIndices) + { + int lastMapIndex = (int)lastFrameIndex; + int goldenMapIndex = (int)goldenFrameIndex; + + if (!slotOccupancy[lastMapIndex]) + { + // Unlike an unused empty slot, the explicitly signaled LAST slot must own a decoded frame before any + // derived mapping can be consumed. libaom rejects the missing reference at this frame-header boundary. + throw new InvalidImageContentException("An AV1 inter frame requests an unavailable LAST reference."); + } + + if (!slotOccupancy[goldenMapIndex]) + { + // GOLDEN is the other explicitly signaled slot and has the same ownership requirement as LAST. + throw new InvalidImageContentException("An AV1 inter frame requests an unavailable GOLDEN reference."); + } + + int currentFrameSortIndex = 1 << (orderHintBitWidth - 1); + int orderHintMask = currentFrameSortIndex - 1; + InlineArray8 referenceInfo = default; + int lastFrameSortIndex = UnavailableSortIndex; + int goldenFrameSortIndex = UnavailableSortIndex; + + for (int mapIndex = 0; mapIndex < Av1Constants.ReferenceFrameCount; mapIndex++) + { + ref ReferenceFrameInfo info = ref referenceInfo[mapIndex]; + info.MapIndex = mapIndex; + info.SortIndex = UnavailableSortIndex; + + if (!slotOccupancy[mapIndex]) + { + // libaom gives absent reference buffers sort index -1. Keeping empty managed slots in the same + // leading partition prevents their stale order hints from participating in temporal selection. + continue; + } + + int difference = (int)slotOrderHints[mapIndex] - (int)currentOrderHint; + + // get_relative_dist folds the unsigned order-hint difference into the signed half-open interval + // [-2^(bits-1), 2^(bits-1)). Adding the half-range makes -1 available as the absence sentinel while valid + // entries sort from zero through the complete modulo domain. + difference = (difference & orderHintMask) - (difference & currentFrameSortIndex); + info.SortIndex = currentFrameSortIndex + difference; + + if (mapIndex == lastMapIndex) + { + lastFrameSortIndex = info.SortIndex; + } + + if (mapIndex == goldenMapIndex) + { + goldenFrameSortIndex = info.SortIndex; + } + } + + if (lastFrameSortIndex >= currentFrameSortIndex) + { + throw new InvalidImageContentException("An AV1 inter frame requests a current or future frame as LAST."); + } + + if (goldenFrameSortIndex >= currentFrameSortIndex) + { + throw new InvalidImageContentException("An AV1 inter frame requests a current or future frame as GOLDEN."); + } + + // libaom sorts first by shifted output order and then by reference-map index. The explicit tie break is + // normative: equal order hints select the highest map index for latest references and the lowest for earliest + // references. Insertion sort is bounded to eight inline entries and does not allocate or require general sort + // infrastructure at the frame-header boundary. + for (int index = 1; index < Av1Constants.ReferenceFrameCount; index++) + { + ReferenceFrameInfo current = referenceInfo[index]; + int insertionIndex = index; + + while (insertionIndex > 0) + { + ReferenceFrameInfo previous = referenceInfo[insertionIndex - 1]; + if (previous.SortIndex < current.SortIndex || + (previous.SortIndex == current.SortIndex && previous.MapIndex <= current.MapIndex)) + { + break; + } + + referenceInfo[insertionIndex] = previous; + insertionIndex--; + } + + referenceInfo[insertionIndex] = current; + } + + InlineArray8 assignedReferences = default; + int lastReferenceIndex = (int)Av1ReferenceFrameType.Last - ReferenceIndexOffset; + int goldenReferenceIndex = (int)Av1ReferenceFrameType.Golden - ReferenceIndexOffset; + referenceFrameIndices[lastReferenceIndex] = lastFrameIndex; + referenceFrameIndices[goldenReferenceIndex] = goldenFrameIndex; + assignedReferences[lastReferenceIndex] = true; + assignedReferences[goldenReferenceIndex] = true; + + int forwardStartIndex = 0; + int forwardEndIndex = Av1Constants.ReferenceFrameCount - 1; + + // Empty entries sort before every occupied shifted hint. The first current-or-future entry then divides the + // remaining sorted table into forward references on the left and backward references on the right. + for (int index = 0; index < Av1Constants.ReferenceFrameCount; index++) + { + if (referenceInfo[index].SortIndex == UnavailableSortIndex) + { + forwardStartIndex++; + continue; + } + + if (referenceInfo[index].SortIndex >= currentFrameSortIndex) + { + forwardEndIndex = index - 1; + break; + } + } + + int backwardStartIndex = forwardEndIndex + 1; + int backwardEndIndex = Av1Constants.ReferenceFrameCount - 1; + int alternateReferenceIndex = (int)Av1ReferenceFrameType.Alternate - ReferenceIndexOffset; + int backwardReferenceIndex = (int)Av1ReferenceFrameType.Backward - ReferenceIndexOffset; + int alternate2ReferenceIndex = (int)Av1ReferenceFrameType.Alternate2 - ReferenceIndexOffset; + + if (backwardStartIndex <= backwardEndIndex) + { + // ALTREF receives the frame farthest into the future. The sorted-map-index tie break selects the highest + // slot when multiple frames share that order hint, matching both the specification and libaom. + referenceFrameIndices[alternateReferenceIndex] = (uint)referenceInfo[backwardEndIndex].MapIndex; + assignedReferences[alternateReferenceIndex] = true; + backwardEndIndex--; + } + + if (backwardStartIndex <= backwardEndIndex) + { + // BWDREF receives the nearest future frame and therefore consumes the low end of the backward partition. + referenceFrameIndices[backwardReferenceIndex] = (uint)referenceInfo[backwardStartIndex].MapIndex; + assignedReferences[backwardReferenceIndex] = true; + backwardStartIndex++; + } + + if (backwardStartIndex <= backwardEndIndex) + { + // ALTREF2 receives the next-nearest remaining future frame. No further backward lookup follows, so the + // lower boundary does not need to advance after this assignment. + referenceFrameIndices[alternate2ReferenceIndex] = (uint)referenceInfo[backwardStartIndex].MapIndex; + assignedReferences[alternate2ReferenceIndex] = true; + } + + ReadOnlySpan remainingReferenceOrder = RemainingReferenceOrder; + int remainingIndex; + + for (remainingIndex = 0; remainingIndex < remainingReferenceOrder.Length; remainingIndex++) + { + int referenceIndex = (int)remainingReferenceOrder[remainingIndex] - ReferenceIndexOffset; + if (assignedReferences[referenceIndex]) + { + continue; + } + + // LAST and GOLDEN were already assigned explicitly and cannot be reused while an unassigned forward slot + // remains. Moving from the high end chooses the remaining frames in anti-chronological order. + while (forwardStartIndex <= forwardEndIndex && + (referenceInfo[forwardEndIndex].MapIndex == lastMapIndex || + referenceInfo[forwardEndIndex].MapIndex == goldenMapIndex)) + { + forwardEndIndex--; + } + + if (forwardStartIndex > forwardEndIndex) + { + break; + } + + referenceFrameIndices[referenceIndex] = (uint)referenceInfo[forwardEndIndex].MapIndex; + assignedReferences[referenceIndex] = true; + forwardEndIndex--; + } + + for (; remainingIndex < remainingReferenceOrder.Length; remainingIndex++) + { + int referenceIndex = (int)remainingReferenceOrder[remainingIndex] - ReferenceIndexOffset; + if (assignedReferences[referenceIndex]) + { + continue; + } + + // AV1 requires every unfilled role to reuse the earliest available forward frame. At least LAST and GOLDEN + // are occupied forward references, so forwardStartIndex always identifies a usable slot at this point. + referenceFrameIndices[referenceIndex] = (uint)referenceInfo[forwardStartIndex].MapIndex; + assignedReferences[referenceIndex] = true; + } + } + + /// + /// Stores one reference-map slot and its shifted order for fixed-size sorting. + /// + private struct ReferenceFrameInfo + { + /// + /// Gets or sets the zero-based slot in the eight-entry persisted reference map. + /// + public int MapIndex { get; set; } + + /// + /// Gets or sets the order hint shifted around the current frame, or when unavailable. + /// + public int SortIndex { get; set; } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameStore.cs b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameStore.cs new file mode 100644 index 000000000..307dde453 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/ReferenceFrames/Av1ReferenceFrameStore.cs @@ -0,0 +1,265 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; + +/// +/// Owns the reference map and selected presentation output for one bounded AV1 decoder session. +/// +/// +/// Several slots and the selected output may identify the same . The store preserves +/// that sharing without allocating reference-count objects and releases a frame only after its final owning reference +/// has been replaced or cleared. This type is not thread safe; one decoder session serializes commit and disposal. +/// +internal sealed class Av1ReferenceFrameStore : IDisposable +{ + /// + /// The number of reference slots defined by the AV1 uncompressed frame header. + /// + private const int SlotCount = Av1Constants.ReferenceFrameCount; + + /// + /// Stores the frame owner selected by each reference-map slot without allocating a managed array. + /// + private InlineArray8 frames; + + /// + /// The most recent shown frame retained for presentation at the end of the bounded image payload. + /// + private Av1ReferenceFrame? outputFrame; + + /// + /// Gets the most recent shown frame retained for presentation. + /// + public Av1ReferenceFrame? OutputFrame => this.outputFrame; + + /// + /// Resolves one reference-map slot. + /// + /// The zero-based reference-map slot in the inclusive range 0 through 7. + /// The retained frame, or when the slot has not been populated. + public Av1ReferenceFrame? Resolve(int slot) => this.frames[slot]; + + /// + /// Writes whether each reference-map slot currently owns a reconstructed frame. + /// + /// The eight-entry destination receiving the current slot occupancy. + public void FillOccupancy(Span destination) + { + // Physical ownership is intentionally independent from frame-ID validity. Short reference signaling sorts + // every occupied slot first, then the uncompressed-header parser validates each derived role separately. + for (int slot = 0; slot < SlotCount; slot++) + { + destination[slot] = this.frames[slot] is not null; + } + } + + /// + /// Commits a completed frame to the reference map and, when shown, retains it for presentation. + /// + /// + /// The mask whose bit n replaces reference-map slot n. Only the low eight bits describe AV1 slots. + /// + /// The completed frame to retain in every selected ownership role. + /// Whether the completed frame replaces the previously retained presentation output. + /// + /// when the frame is retained as a reference or presentation output and ownership transfers + /// to this store; otherwise , in which case no state changes and the caller retains ownership. + /// + /// + /// The caller must invoke this method only after reconstruction and all normative in-loop filters have completed. + /// Once ownership transfers, the caller must not dispose the frame. A frame passed here must not already be owned by + /// this store. + /// + public bool Commit(uint refreshFrameFlags, Av1ReferenceFrame frame, bool showFrame) + { + refreshFrameFlags &= byte.MaxValue; + if (refreshFrameFlags == 0 && !showFrame) + { + // A hidden frame with a zero refresh mask has no remaining role in an image-decoder session. + return false; + } + + InlineArray8 replacedFrames = default; + Av1ReferenceFrame? replacedOutputFrame = showFrame ? this.outputFrame : null; + + // Capture displaced owners in inline storage, then publish the complete slot and output transition before + // releasing anything. A shown frame may also occupy reference slots, so both ownership domains must change as + // one operation. + for (int slot = 0; slot < SlotCount; slot++) + { + if ((refreshFrameFlags & (1U << slot)) != 0) + { + replacedFrames[slot] = this.frames[slot]; + this.frames[slot] = frame; + } + } + + if (showFrame) + { + this.outputFrame = frame; + } + + for (int replacedIndex = 0; replacedIndex < SlotCount; replacedIndex++) + { + Av1ReferenceFrame? replacedFrame = replacedFrames[replacedIndex]; + + if (replacedFrame is null) + { + continue; + } + + if (ReferenceEquals(replacedFrame, replacedOutputFrame)) + { + // Let the displaced-output path release this shared owner after every slot candidate has been removed. + replacedFrames[replacedIndex] = null; + continue; + } + + // A displaced frame remains owned when any unrefreshed slot or the selected output still references it. + // Eight fixed slots make the bounded identity scan cheaper than allocated reference-count state. + if (this.IsRetained(replacedFrame)) + { + replacedFrames[replacedIndex] = null; + } + } + + DisposeUnique(ref replacedFrames); + + if (replacedOutputFrame is not null && !this.IsRetained(replacedOutputFrame)) + { + replacedOutputFrame.Dispose(); + } + + return true; + } + + /// + /// Replaces the selected presentation output with an independently owned completed frame. + /// + /// The completed presentation frame whose ownership transfers to this store. + /// + /// This path is used when film grain requires presentation samples to differ from the ungrained reconstruction + /// retained by the reference map. + /// + public void CommitOutput(Av1ReferenceFrame frame) + { + Av1ReferenceFrame? replacedFrame = this.outputFrame; + this.outputFrame = frame; + + // The previous output may still be retained by one or more reference slots. Release it only after publishing + // the new output and confirming that no reference-map identity remains. + if (replacedFrame is not null && !this.IsRetained(replacedFrame)) + { + replacedFrame.Dispose(); + } + } + + /// + /// Transfers the selected presentation frame out of this store and releases every other retained frame. + /// + /// The selected presentation frame now owned by the caller. + public Av1ReferenceFrame TakeOutput() + { + Av1ReferenceFrame result = this.outputFrame!; + this.outputFrame = null; + + // The caller becomes the sole owner of the selected output. Remove all slot aliases before Reset releases the + // remaining session references so the sample buffer can transfer without copying. + for (int slot = 0; slot < SlotCount; slot++) + { + if (ReferenceEquals(this.frames[slot], result)) + { + this.frames[slot] = null; + } + } + + this.Reset(); + return result; + } + + /// + /// Clears all reference-map slots and releases every uniquely retained frame. + /// + public void Reset() + { + InlineArray8 releasedFrames = this.frames; + this.frames = default; + Av1ReferenceFrame? releasedOutputFrame = this.outputFrame; + this.outputFrame = null; + + // Clear the live map before disposal so the store cannot expose a partially reset ownership state. When the + // output aliases a slot, let the output path perform the single release after the duplicate slot is removed. + if (releasedOutputFrame is not null) + { + for (int slot = 0; slot < SlotCount; slot++) + { + if (ReferenceEquals(releasedFrames[slot], releasedOutputFrame)) + { + releasedFrames[slot] = null; + } + } + } + + DisposeUnique(ref releasedFrames); + releasedOutputFrame?.Dispose(); + } + + /// + /// Releases every uniquely retained frame and clears all reference-map slots. + /// + public void Dispose() => this.Reset(); + + /// + /// Determines whether the live reference map or presentation output retains a frame. + /// + /// The frame whose ownership is queried. + /// when the store still owns the frame. + private bool IsRetained(Av1ReferenceFrame frame) + { + if (ReferenceEquals(this.outputFrame, frame)) + { + return true; + } + + for (int slot = 0; slot < SlotCount; slot++) + { + if (ReferenceEquals(this.frames[slot], frame)) + { + return true; + } + } + + return false; + } + + /// + /// Releases each distinct frame owner in a fixed-size set exactly once. + /// + /// The inline set of frame references to release. + private static void DisposeUnique(ref InlineArray8 frames) + { + for (int frameIndex = 0; frameIndex < SlotCount; frameIndex++) + { + Av1ReferenceFrame? frame = frames[frameIndex]; + + if (frame is null) + { + continue; + } + + // Null every later alias before disposal. The store intentionally represents shared slot ownership through + // object identity, so no separately allocated reference-count state is needed for the eight-entry map. + for (int duplicateIndex = frameIndex + 1; duplicateIndex < SlotCount; duplicateIndex++) + { + if (ReferenceEquals(frames[duplicateIndex], frame)) + { + frames[duplicateIndex] = null; + } + } + + frames[frameIndex] = null; + frame.Dispose(); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs index b81ebf44f..edeb939d5 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1BlockModeInfo.cs @@ -1,16 +1,33 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// -/// Stores block-size, prediction-mode, transform, and palette decisions shared by AV1 block processing. +/// Stores block-size, intra/inter prediction, transform, and palette decisions shared by AV1 block processing. /// internal class Av1BlockModeInfo { + /// + /// Stores the primary and optional secondary reference-frame labels. + /// + private InlineArray2 referenceFrames; + + /// + /// Stores the motion vector associated with each reference-frame label. + /// + private InlineArray2 motionVectors; + + /// + /// Stores the vertical and horizontal subpixel interpolation filters in that order. + /// + private InlineArray2 interpolationFilters; + /// /// The palette size for the luma plane. /// @@ -85,6 +102,11 @@ internal class Av1BlockModeInfo { this.BlockSize = blockSize; this.PositionInSuperblock = positionInSuperblock; + + // Both entries begin absent because inter syntax has not selected either reference yet. Intra parsing replaces + // the primary entry with the current frame while retaining None as the optional secondary reference. + this.referenceFrames[0] = Av1ReferenceFrameType.None; + this.referenceFrames[1] = Av1ReferenceFrameType.None; } /// @@ -97,6 +119,89 @@ internal class Av1BlockModeInfo /// public Av1PredictionMode YMode { get; set; } + /// + /// Gets the primary and optional secondary reference-frame labels. + /// + /// + /// Index zero is the primary reference. Index one is for a single-reference + /// block, for an inter-intra block, or the secondary inter-reference label + /// for compound prediction. + /// + public Span ReferenceFrames => this.referenceFrames; + + /// + /// Gets the decoded motion vectors corresponding to . + /// + public Span MotionVectors => this.motionVectors; + + /// + /// Gets the interpolation filters used for vertical and horizontal subpixel prediction. + /// + /// + /// Index zero is the vertical filter and index one is the horizontal filter, matching libaom's + /// InterpFilters.y_filter and InterpFilters.x_filter layout. + /// + public Span InterpolationFilters => this.interpolationFilters; + + /// + /// Gets or sets the selected index in the derived reference-motion-vector stack. + /// + /// + /// The AV1 syntax constrains this value to the inclusive range 0 through 3. + /// + public byte ReferenceMotionVectorIndex { get; set; } + + /// + /// Gets or sets the motion model used to construct inter prediction. + /// + public Av1MotionMode MotionMode { get; set; } + + /// + /// Gets or sets the intra predictor blended with a single-reference inter predictor. + /// + public Av1InterIntraMode InterIntraMode { get; set; } + + /// + /// Gets or sets a value indicating whether inter-intra prediction uses a wedge mask. + /// + public bool UseInterIntraWedge { get; set; } + + /// + /// Gets or sets the inter-intra wedge-mask index in the inclusive range 0 through 15. + /// + public byte InterIntraWedgeIndex { get; set; } + + /// + /// Gets or sets a value indicating whether compound prediction uses the masked-compound mode group. + /// + public bool CompoundGroupIndex { get; set; } + + /// + /// Gets or sets a value indicating whether unmasked compound prediction uses average blending. + /// A value of selects distance-weighted blending. + /// + public bool CompoundIndex { get; set; } + + /// + /// Gets or sets the compound blending method selected for two inter predictors. + /// + public Av1CompoundType CompoundType { get; set; } + + /// + /// Gets or sets the compound wedge-mask index in the inclusive range 0 through 15. + /// + public byte CompoundWedgeIndex { get; set; } + + /// + /// Gets or sets a value indicating whether the compound wedge mask is inverted. + /// + public bool CompoundWedgeSign { get; set; } + + /// + /// Gets or sets the orientation of the difference-weighted compound mask. + /// + public Av1DifferenceWeightedMaskType DifferenceWeightedMaskType { get; set; } + /// /// Gets or sets a value indicating whether residual coefficients are omitted for the block. /// @@ -118,9 +223,9 @@ internal class Av1BlockModeInfo public int SegmentId { get; set; } /// - /// Gets or sets the for the chroma channels. + /// Gets or sets the chroma intra-prediction mode. /// - public Av1PredictionMode UvMode { get; set; } + public Av1ChromaPredictionMode UvMode { get; set; } /// /// Gets or sets a value indicating whether intra block copy is selected. @@ -334,4 +439,17 @@ internal class Av1BlockModeInfo this.chromaPaletteColorIndexMap = colorIndexMap; } } + + /// + /// Provides fixed storage for the two values associated with AV1's primary and secondary inter references. + /// + /// The stored reference label, motion vector, or interpolation-filter type. + [InlineArray(2)] + private struct InlineArray2 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1CompoundType.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1CompoundType.cs new file mode 100644 index 000000000..35c4c71f6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1CompoundType.cs @@ -0,0 +1,30 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +/// +/// Identifies the blending method used to combine two AV1 inter predictors. +/// +internal enum Av1CompoundType : byte +{ + /// + /// Averages both predictors with equal weights. + /// + Average = 0, + + /// + /// Weights predictors from their relative display-order distances. + /// + DistanceWeighted = 1, + + /// + /// Selects per-pixel weights from a signaled wedge mask. + /// + Wedge = 2, + + /// + /// Derives per-pixel weights from the difference between both predictors. + /// + DifferenceWeighted = 3, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1DifferenceWeightedMaskType.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1DifferenceWeightedMaskType.cs new file mode 100644 index 000000000..81dbcfa70 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1DifferenceWeightedMaskType.cs @@ -0,0 +1,20 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +/// +/// Identifies the orientation of an AV1 difference-weighted compound mask. +/// +internal enum Av1DifferenceWeightedMaskType : byte +{ + /// + /// Applies the predictor-difference adjustment to a base alpha weight of 38 on AV1's 0-through-64 blend scale. + /// + Type38 = 0, + + /// + /// Applies the complement of the type-38 predictor-difference mask. + /// + Type38Inverse = 1, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs index 949e5f23c..c97e4a582 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs @@ -28,7 +28,7 @@ internal class Av1EncoderBlockModeInfo /// /// Gets the selected chroma prediction mode. /// - public Av1PredictionMode UvPredictionMode { get; } + public Av1ChromaPredictionMode UvPredictionMode { get; } /// /// Gets a value indicating whether residual coefficients are omitted for the block. @@ -63,5 +63,5 @@ internal class Av1EncoderBlockModeInfo /// /// Gets or sets the chroma prediction mode written for the block. /// - public Av1PredictionMode UvMode { get; internal set; } + public Av1ChromaPredictionMode UvMode { get; internal set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.MotionField.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.MotionField.cs new file mode 100644 index 000000000..d68a63e9c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.MotionField.cs @@ -0,0 +1,536 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +/// +/// Owns the retained and projected per-8x8 motion fields associated with a decoded AV1 frame. +/// +internal partial class Av1FrameInfo +{ + /// + /// The maximum absolute temporal distance accepted by AV1 motion-field projection. + /// + private const int MaximumFrameDistance = 31; + + /// + /// The maximum number of reference frames projected into one temporal motion field. + /// + private const int MotionFieldProjectionCount = 3; + + /// + /// The maximum source motion-vector magnitude retained for later temporal projection. + /// + private const int ReferenceMotionVectorLimit = 4095; + + /// + /// The exclusive upper bound of an AV1 motion-vector component in one-eighth-sample units. + /// + private const int MotionVectorUpperBound = 16384; + + /// + /// The reserved lower endpoint of an AV1 motion-vector component in one-eighth-sample units. + /// + private const int MotionVectorLowerBound = -16384; + + /// + /// The width or height of the largest AV1 superblock in 4x4 mode-information units. + /// + private const int MaximumSuperblockModeInfoSize = 1 << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2); + + /// + /// The base-two logarithm of . + /// + private const int MaximumSuperblockModeInfoSizeLog2 = Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2; + + /// + /// The base-two reduction from 4x4 mode-information coordinates to the 8x8 motion-field grid. + /// + private const int MotionFieldModeInfoShift = 1; + + /// + /// The base-two reduction from one-eighth-sample motion vectors to offsets on the 8x8 motion-field grid. + /// + private const int MotionVectorToFieldOffsetShift = 4 + Av1Constants.ModeInfoSizeLog2; + + /// + /// The maximum horizontal projection displacement, measured in 8x8 motion-field blocks. + /// + private const int MaximumHorizontalFieldOffset = 8; + + /// + /// Stores the selected motion vector and logical reference for every retained 8x8 frame position. + /// + private RetainedMotionFieldEntry[] retainedMotionField = []; + + /// + /// Stores motion vectors projected from retained frames into the current frame's 8x8 grid. + /// + private TemporalMotionFieldEntry[] temporalMotionField = []; + + /// + /// Stores the order hint selected by each logical inter-reference type for later projections from this frame. + /// + private InlineArray8 motionFieldReferenceOrderHints; + + /// + /// Stores whether each logical inter-reference type lies after, at, or before the current frame in display order. + /// + private InlineArray8 motionFieldReferenceSides; + + /// + /// The number of retained motion-field entries in one active 8x8 row. + /// + private int retainedMotionFieldStride; + + /// + /// The number of projected temporal-motion entries in one aligned 8x8 row. + /// + private int temporalMotionFieldStride; + + /// + /// The active frame width in 4x4 mode-information units. + /// + private int activeModeInfoColumnCount; + + /// + /// The active frame height in 4x4 mode-information units. + /// + private int activeModeInfoRowCount; + + /// + /// Gets the reciprocal table used by AV1 motion-vector projection in 14-bit fixed-point precision. + /// + private static ReadOnlySpan ProjectionDivisors => + [0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, + 1024, 963, 910, 862, 819, 780, 744, 712, 682, 655, 630, 606, 585, 564, 546, 528]; + + /// + /// Allocates and derives the motion fields required by one decoded frame. + /// + /// The sequence header defining motion-field enablement and order-hint precision. + /// The current frame header and its seven resolved inter-reference roles. + /// The retained reconstructed frames selected by the current reference map. + public void InitializeMotionField( + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1ReferenceFrameStore referenceFrames) + { + if (!sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors) + { + return; + } + + this.activeModeInfoColumnCount = frameHeader.ModeInfoColumnCount; + this.activeModeInfoRowCount = frameHeader.ModeInfoRowCount; + this.retainedMotionFieldStride = (this.activeModeInfoColumnCount + 1) >> MotionFieldModeInfoShift; + + if (frameHeader.IsIntra) + { + // Intra frames retain an empty source field. They can occupy reference slots, but libaom rejects them as + // projection sources before consulting their reference-order-hint metadata. + return; + } + + InlineArray8 selectedReferences = default; + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + int orderHintBits = sequenceHeader.OrderHintInfo.OrderHintBits; + + // Capture the seven logical-role order hints before this frame refreshes any physical map slots. Libaom keeps + // the same snapshot on RefCntBuffer so a later frame can project this frame's stored motion vectors. + for (int referenceIndex = 0; referenceIndex < Av1Constants.ReferencesPerFrame; referenceIndex++) + { + Av1ReferenceFrameType referenceFrameType = (Av1ReferenceFrameType)(referenceIndex + 1); + Av1ReferenceFrame referenceFrame = referenceFrames.Resolve((int)referenceFrameIndices[referenceIndex])!; + uint referenceOrderHint = referenceFrame.FrameHeader.OrderHint; + selectedReferences[(int)referenceFrameType] = referenceFrame; + this.motionFieldReferenceOrderHints[(int)referenceFrameType] = referenceOrderHint; + + int relativeDistance = GetRelativeDistance(referenceOrderHint, frameHeader.OrderHint, orderHintBits); + this.motionFieldReferenceSides[(int)referenceFrameType] = relativeDistance > 0 + ? (sbyte)1 + : referenceOrderHint == frameHeader.OrderHint ? (sbyte)-1 : (sbyte)0; + } + + // FrameInfo is transferred directly into each retained frame owner, so allocate only the active 8x8 source + // grid whose completed block vectors can be projected by a later frame. + int retainedRowCount = (this.activeModeInfoRowCount + 1) >> MotionFieldModeInfoShift; + this.retainedMotionField = new RetainedMotionFieldEntry[this.retainedMotionFieldStride * retainedRowCount]; + + if (!frameHeader.UseReferenceFrameMotionVectors) + { + return; + } + + // libaom aligns the projected field stride to the largest superblock even for a 64x64 sequence. This keeps + // later temporal-candidate addressing independent of the current sequence's selected superblock size. + int alignedModeInfoColumnCount = Av1Math.AlignPowerOf2( + this.activeModeInfoColumnCount, + MaximumSuperblockModeInfoSizeLog2); + + this.temporalMotionFieldStride = alignedModeInfoColumnCount >> MotionFieldModeInfoShift; + int temporalRowCount = (this.activeModeInfoRowCount + MaximumSuperblockModeInfoSize) >> MotionFieldModeInfoShift; + this.temporalMotionField = new TemporalMotionFieldEntry[this.temporalMotionFieldStride * temporalRowCount]; + + // AV1 examines LAST, BWDREF, ALTREF2, ALTREF, and LAST2 in this normative order and admits at most three + // projection sources. LAST always consumes the first budget position, forward references consume one only + // when eligible projection succeeds, and LAST2 fills the final unused position in the reverse direction. + int remainingProjectionCount = MotionFieldProjectionCount; + Av1ReferenceFrame lastFrame = selectedReferences[(int)Av1ReferenceFrameType.Last]!; + Av1ReferenceFrame goldenFrame = selectedReferences[(int)Av1ReferenceFrameType.Golden]!; + uint alternateOfLastOrderHint = lastFrame.FrameInfo.motionFieldReferenceOrderHints[(int)Av1ReferenceFrameType.Alternate]; + + // A LAST frame whose ALTREF order matches GOLDEN is an overlay. Projecting it would duplicate the overlay's + // temporal source, but libaom still consumes one position from the three-source projection budget. + if (alternateOfLastOrderHint != goldenFrame.FrameHeader.OrderHint) + { + _ = this.ProjectMotionField(sequenceHeader, frameHeader, lastFrame, reverseDirection: true); + } + + remainingProjectionCount--; + Av1ReferenceFrame backwardFrame = selectedReferences[(int)Av1ReferenceFrameType.Backward]!; + + if (GetRelativeDistance(backwardFrame.FrameHeader.OrderHint, frameHeader.OrderHint, orderHintBits) > 0 && + this.ProjectMotionField(sequenceHeader, frameHeader, backwardFrame, reverseDirection: false)) + { + remainingProjectionCount--; + } + + Av1ReferenceFrame alternate2Frame = selectedReferences[(int)Av1ReferenceFrameType.Alternate2]!; + + if (GetRelativeDistance(alternate2Frame.FrameHeader.OrderHint, frameHeader.OrderHint, orderHintBits) > 0 && + this.ProjectMotionField(sequenceHeader, frameHeader, alternate2Frame, reverseDirection: false)) + { + remainingProjectionCount--; + } + + Av1ReferenceFrame alternateFrame = selectedReferences[(int)Av1ReferenceFrameType.Alternate]!; + + if (remainingProjectionCount > 0 && + GetRelativeDistance(alternateFrame.FrameHeader.OrderHint, frameHeader.OrderHint, orderHintBits) > 0 && + this.ProjectMotionField(sequenceHeader, frameHeader, alternateFrame, reverseDirection: false)) + { + remainingProjectionCount--; + } + + if (remainingProjectionCount > 0) + { + Av1ReferenceFrame last2Frame = selectedReferences[(int)Av1ReferenceFrameType.Last2]!; + _ = this.ProjectMotionField(sequenceHeader, frameHeader, last2Frame, reverseDirection: true); + } + } + + /// + /// Gets the temporal motion vector projected over a 4x4 mode-information position. + /// + /// The zero-based 4x4 row. + /// The zero-based 4x4 column. + /// Receives the retained source vector in one-eighth-sample units. + /// Receives the positive temporal distance from the source to its reference. + /// when a projected vector covers the requested position. + public bool TryGetTemporalMotionVector( + int modeInfoRow, + int modeInfoColumn, + out Av1MotionVector motionVector, + out int referenceFrameOffset) + { + int index = ((modeInfoRow >> MotionFieldModeInfoShift) * this.temporalMotionFieldStride) + + (modeInfoColumn >> MotionFieldModeInfoShift); + + TemporalMotionFieldEntry entry = this.temporalMotionField[index]; + motionVector = entry.MotionVector; + referenceFrameOffset = entry.ReferenceFrameOffset; + return referenceFrameOffset > 0; + } + + /// + /// Writes the retained per-8x8 motion-field entries covered by one completed mode-information block. + /// + /// The completed block mode information. + /// The block origin in frame-relative 4x4 units. + private void UpdateRetainedMotionField(Av1BlockModeInfo modeInfo, Point modeInfoPosition) + { + if (this.retainedMotionField.Length == 0) + { + return; + } + + Av1ReferenceFrameType selectedReference = Av1ReferenceFrameType.None; + Av1MotionVector selectedMotionVector = default; + Span referenceFrames = modeInfo.ReferenceFrames; + Span motionVectors = modeInfo.MotionVectors; + + // Compound blocks may offer two vectors. libaom retains the last eligible forward-or-past reference after + // excluding same-order, future, and out-of-range vectors, so preserve that overwrite order exactly. + for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++) + { + Av1ReferenceFrameType referenceFrame = referenceFrames[referenceIndex]; + Av1MotionVector motionVector = motionVectors[referenceIndex]; + + if (referenceFrame > Av1ReferenceFrameType.Intra && + this.motionFieldReferenceSides[(int)referenceFrame] == 0 && + Math.Abs(motionVector.Row) <= ReferenceMotionVectorLimit && + Math.Abs(motionVector.Column) <= ReferenceMotionVectorLimit) + { + selectedReference = referenceFrame; + selectedMotionVector = motionVector; + } + } + + int blockModeInfoWidth = Math.Min( + modeInfo.BlockSize.Get4x4WideCount(), + this.activeModeInfoColumnCount - modeInfoPosition.X); + + int blockModeInfoHeight = Math.Min( + modeInfo.BlockSize.Get4x4HighCount(), + this.activeModeInfoRowCount - modeInfoPosition.Y); + + int fieldWidth = (blockModeInfoWidth + 1) >> MotionFieldModeInfoShift; + int fieldHeight = (blockModeInfoHeight + 1) >> MotionFieldModeInfoShift; + int firstFieldRow = modeInfoPosition.Y >> MotionFieldModeInfoShift; + int firstFieldColumn = modeInfoPosition.X >> MotionFieldModeInfoShift; + RetainedMotionFieldEntry entry = new(selectedMotionVector, selectedReference); + + // One decoded block supplies the same retained candidate to every covered 8x8 cell. Array.Fill preserves the + // native contiguous-row write and lets later sub-8x8 blocks overwrite the shared cell in traversal order. + for (int row = 0; row < fieldHeight; row++) + { + int rowOffset = ((firstFieldRow + row) * this.retainedMotionFieldStride) + firstFieldColumn; + Array.Fill(this.retainedMotionField, entry, rowOffset, fieldWidth); + } + } + + /// + /// Projects one retained frame's motion field into the current frame's temporal candidate grid. + /// + /// The sequence header defining the modulo order-hint domain. + /// The current frame header. + /// The retained frame whose stored motion vectors are projected. + /// + /// A value indicating whether the start-to-current distance and spatial displacement are reversed for a past frame. + /// + /// when the retained frame is an eligible projection source. + private bool ProjectMotionField( + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1ReferenceFrame startFrame, + bool reverseDirection) + { + ObuFrameHeader startFrameHeader = startFrame.FrameHeader; + if (startFrameHeader.IsIntra || + startFrameHeader.ModeInfoRowCount != this.activeModeInfoRowCount || + startFrameHeader.ModeInfoColumnCount != this.activeModeInfoColumnCount) + { + // AV1 does not rescale temporal fields. Intra sources contain no inter motion, and a differently sized + // source has no one-to-one 8x8 grid on which the normative projection can operate. + return false; + } + + Av1FrameInfo startFrameInfo = startFrame.FrameInfo; + int orderHintBits = sequenceHeader.OrderHintInfo.OrderHintBits; + int startToCurrentFrameOffset = GetRelativeDistance( + startFrameHeader.OrderHint, + frameHeader.OrderHint, + orderHintBits); + + if (reverseDirection) + { + startToCurrentFrameOffset = -startToCurrentFrameOffset; + } + + int sourceRowCount = (this.activeModeInfoRowCount + 1) >> MotionFieldModeInfoShift; + int sourceColumnCount = (this.activeModeInfoColumnCount + 1) >> MotionFieldModeInfoShift; + int destinationRowCount = this.activeModeInfoRowCount >> MotionFieldModeInfoShift; + int destinationColumnCount = this.activeModeInfoColumnCount >> MotionFieldModeInfoShift; + + for (int blockRow = 0; blockRow < sourceRowCount; blockRow++) + { + int sourceRowOffset = blockRow * startFrameInfo.retainedMotionFieldStride; + for (int blockColumn = 0; blockColumn < sourceColumnCount; blockColumn++) + { + RetainedMotionFieldEntry source = startFrameInfo.retainedMotionField[sourceRowOffset + blockColumn]; + if (source.ReferenceFrame <= Av1ReferenceFrameType.Intra) + { + continue; + } + + int referenceFrameOffset = GetRelativeDistance( + startFrameHeader.OrderHint, + startFrameInfo.motionFieldReferenceOrderHints[(int)source.ReferenceFrame], + orderHintBits); + + bool positionIsValid = Math.Abs(referenceFrameOffset) <= MaximumFrameDistance && + referenceFrameOffset > 0 && + Math.Abs(startToCurrentFrameOffset) <= MaximumFrameDistance; + + if (!positionIsValid) + { + continue; + } + + Av1MotionVector projected = ProjectMotionVector( + source.MotionVector, + startToCurrentFrameOffset, + referenceFrameOffset); + + if (!TryGetProjectedBlockPosition( + blockRow, + blockColumn, + projected, + reverseDirection, + destinationRowCount, + destinationColumnCount, + out int projectedRow, + out int projectedColumn)) + { + continue; + } + + // The projected vector selects the destination cell, but AV1 stores the original forward vector and + // its source-to-reference distance there. Candidate scaling later uses both values for its own target. + int destinationOffset = (projectedRow * this.temporalMotionFieldStride) + projectedColumn; + this.temporalMotionField[destinationOffset] = new(source.MotionVector, referenceFrameOffset); + } + } + + return true; + } + + /// + /// Scales a retained motion vector by a signed ratio of temporal distances. + /// + /// The retained vector in one-eighth-sample units. + /// The signed start-to-current temporal distance. + /// The positive start-to-reference temporal distance. + /// The projected and AV1-range-clamped vector. + private static Av1MotionVector ProjectMotionVector(Av1MotionVector motionVector, int numerator, int denominator) + { + denominator = Math.Min(denominator, MaximumFrameDistance); + numerator = Av1Math.Clip3(-MaximumFrameDistance, MaximumFrameDistance, numerator); + + // ProjectionDivisors represents 1 / denominator in Q14. Symmetric power-of-two rounding matches libaom for + // negative vectors, and the final clamp excludes the two reserved extreme motion-vector values. + int row = Av1Math.RoundPowerOf2Signed(motionVector.Row * numerator * ProjectionDivisors[denominator], 14); + int column = Av1Math.RoundPowerOf2Signed(motionVector.Column * numerator * ProjectionDivisors[denominator], 14); + row = Av1Math.Clip3(MotionVectorLowerBound + 1, MotionVectorUpperBound - 1, row); + column = Av1Math.Clip3(MotionVectorLowerBound + 1, MotionVectorUpperBound - 1, column); + return new(row, column); + } + + /// + /// Maps a projected motion vector to its bounded destination on the current 8x8 field. + /// + /// The source 8x8 row. + /// The source 8x8 column. + /// The temporally projected vector in one-eighth-sample units. + /// Whether the vector moves backwards from the source position. + /// The number of complete 8x8 rows in the current frame. + /// The number of complete 8x8 columns in the current frame. + /// Receives the projected 8x8 row. + /// Receives the projected 8x8 column. + /// when the destination lies in the permitted projection window. + private static bool TryGetProjectedBlockPosition( + int blockRow, + int blockColumn, + Av1MotionVector motionVector, + bool reverseDirection, + int rowCount, + int columnCount, + out int projectedRow, + out int projectedColumn) + { + int baseBlockRow = (blockRow >> 3) << 3; + int baseBlockColumn = (blockColumn >> 3) << 3; + + // One field cell spans 8 samples, while vectors use one-eighth-sample units; dividing by 64 converts between + // them. C# integer division truncates toward zero, matching libaom's explicit signed-shift construction. + int rowOffset = motionVector.Row / (1 << MotionVectorToFieldOffsetShift); + int columnOffset = motionVector.Column / (1 << MotionVectorToFieldOffsetShift); + projectedRow = reverseDirection ? blockRow - rowOffset : blockRow + rowOffset; + projectedColumn = reverseDirection ? blockColumn - columnOffset : blockColumn + columnOffset; + + if (projectedRow < 0 || projectedRow >= rowCount || projectedColumn < 0 || projectedColumn >= columnCount) + { + return false; + } + + // AV1 keeps a projection in the same 64x64 row band and permits one additional 64-sample horizontal band on + // either side. This bounds temporal-candidate lookup while accommodating common lateral motion. + return projectedRow >= baseBlockRow && + projectedRow < baseBlockRow + 8 && + projectedColumn >= baseBlockColumn - MaximumHorizontalFieldOffset && + projectedColumn < baseBlockColumn + 8 + MaximumHorizontalFieldOffset; + } + + /// + /// Computes the signed distance between two order hints in their modulo domain. + /// + /// The first order hint. + /// The order hint subtracted from . + /// The number of bits in the order-hint domain. + /// The shortest signed modulo distance. + private static int GetRelativeDistance(uint first, uint second, int orderHintBits) + { + int difference = (int)first - (int)second; + int signBit = 1 << (orderHintBits - 1); + return (difference & (signBit - 1)) - (difference & signBit); + } + + /// + /// Stores one motion vector and logical reference retained for projection by a later frame. + /// + private readonly struct RetainedMotionFieldEntry + { + /// + /// Initializes a new instance of the struct. + /// + /// The retained motion vector in one-eighth-sample units. + /// The logical reference targeted by the vector. + public RetainedMotionFieldEntry(Av1MotionVector motionVector, Av1ReferenceFrameType referenceFrame) + { + this.MotionVector = motionVector; + this.ReferenceFrame = referenceFrame; + } + + /// + /// Gets the retained motion vector in one-eighth-sample units. + /// + public Av1MotionVector MotionVector { get; } + + /// + /// Gets the logical reference targeted by . + /// + public Av1ReferenceFrameType ReferenceFrame { get; } + } + + /// + /// Stores one temporal candidate projected over the current frame's 8x8 grid. + /// + private readonly struct TemporalMotionFieldEntry + { + /// + /// Initializes a new instance of the struct. + /// + /// The retained source vector in one-eighth-sample units. + /// The positive temporal distance from the source to its reference. + public TemporalMotionFieldEntry(Av1MotionVector motionVector, int referenceFrameOffset) + { + this.MotionVector = motionVector; + this.ReferenceFrameOffset = referenceFrameOffset; + } + + /// + /// Gets the retained source vector in one-eighth-sample units. + /// + public Av1MotionVector MotionVector { get; } + + /// + /// Gets the positive temporal distance from the source frame to its reference. + /// + public int ReferenceFrameOffset { get; } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs index 92744b187..a20f0aa79 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs @@ -6,7 +6,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// -/// Owns the mode, transform, coefficient, quantizer, and filter state decoded for one AV1 frame. +/// Owns the mode, motion, segmentation, transform, coefficient, quantizer, and filter state decoded for one AV1 frame. /// internal partial class Av1FrameInfo { @@ -70,6 +70,21 @@ internal partial class Av1FrameInfo /// private readonly Av1FrameModeInfoMap modeInfoMap; + /// + /// Stores the decoded segment identifier for each active 4x4 mode-information position in row-major order. + /// + private byte[] segmentIds = []; + + /// + /// The number of active 4x4 columns in one row of . + /// + private int segmentIdColumnCount; + + /// + /// The number of active 4x4 rows represented by . + /// + private int segmentIdRowCount; + /// /// Stores luma transform information grouped by superblock. /// @@ -188,6 +203,102 @@ internal partial class Av1FrameInfo /// public int SuperblockModeInfoSize => this.modeInfoSizePerSuperblock; + /// + /// Initializes the active frame's contiguous segment map and applies whole-map inheritance when requested. + /// + /// The frame header defining active geometry and segmentation update behavior. + /// + /// The retained state selected by the primary reference, or when no primary reference exists. + /// + public void InitializeSegmentIds(ObuFrameHeader frameHeader, Av1FrameInfo? primaryReferenceFrameInfo) + { + ObuSegmentationParameters segmentationParameters = frameHeader.SegmentationParameters; + if (!segmentationParameters.Enabled) + { + // A disabled map is normatively all zero. Empty storage represents that state without retaining one byte + // for every 4x4 position on frames that cannot use segmentation. + return; + } + + this.segmentIdColumnCount = frameHeader.ModeInfoColumnCount; + this.segmentIdRowCount = frameHeader.ModeInfoRowCount; + this.segmentIds = new byte[this.segmentIdColumnCount * this.segmentIdRowCount]; + + if (segmentationParameters.SegmentationUpdateMap == 0 && + primaryReferenceFrameInfo is not null && + primaryReferenceFrameInfo.segmentIdColumnCount == this.segmentIdColumnCount && + primaryReferenceFrameInfo.segmentIdRowCount == this.segmentIdRowCount) + { + // AV1 decodemv.c copies the selected primary frame's block coverage when update_map is zero. Copying the + // same contiguous map once establishes the identical final state without repeating a row copy per block. + primaryReferenceFrameInfo.segmentIds.CopyTo(this.segmentIds, 0); + } + } + + /// + /// Gets the segment identifier stored at one active 4x4 mode-information position. + /// + /// The zero-based mode-information row. + /// The zero-based mode-information column. + /// The segment identifier stored at the requested position. + public byte GetSegmentId(int row, int column) => this.segmentIds[(row * this.segmentIdColumnCount) + column]; + + /// + /// Gets the minimum retained segment identifier across a block's clipped mode-information coverage. + /// + /// The block size whose 4x4 coverage is inspected. + /// The block origin in frame-relative 4x4 units. + /// + /// The minimum retained segment identifier, or zero when the retained frame has no enabled segmentation map. + /// + public int GetPredictedSegmentId(Av1BlockSize blockSize, Point modeInfoPosition) + { + if (this.segmentIds.Length == 0) + { + // libaom represents an unavailable prior map with a null pointer and predicts segment zero. + return 0; + } + + int columnCount = Math.Min(blockSize.Get4x4WideCount(), this.segmentIdColumnCount - modeInfoPosition.X); + int rowCount = Math.Min(blockSize.Get4x4HighCount(), this.segmentIdRowCount - modeInfoPosition.Y); + int segmentId = Av1Constants.MaxSegmentCount; + + // Temporal prediction uses the minimum over every clipped 4x4 cell, not merely the block origin. This is the + // dec_get_segment_id rule used when segmentation_temporal_update selects the retained primary map. + for (int row = 0; row < rowCount; row++) + { + int offset = ((modeInfoPosition.Y + row) * this.segmentIdColumnCount) + modeInfoPosition.X; + ReadOnlySpan segmentRow = this.segmentIds.AsSpan(offset, columnCount); + + for (int column = 0; column < segmentRow.Length; column++) + { + segmentId = Math.Min(segmentId, segmentRow[column]); + } + } + + return segmentId; + } + + /// + /// Writes one segment identifier over a block's clipped mode-information coverage. + /// + /// The block size whose 4x4 coverage is updated. + /// The block origin in frame-relative 4x4 units. + /// The decoded segment identifier. + public void SetSegmentId(Av1BlockSize blockSize, Point modeInfoPosition, int segmentId) + { + int columnCount = Math.Min(blockSize.Get4x4WideCount(), this.segmentIdColumnCount - modeInfoPosition.X); + int rowCount = Math.Min(blockSize.Get4x4HighCount(), this.segmentIdRowCount - modeInfoPosition.Y); + + // Each block contributes one ID to all covered 4x4 cells. Filling contiguous row slices retains the native + // row-major layout without the per-row object indirection of the previous jagged map. + for (int row = 0; row < rowCount; row++) + { + int offset = ((modeInfoPosition.Y + row) * this.segmentIdColumnCount) + modeInfoPosition.X; + this.segmentIds.AsSpan(offset, columnCount).Fill((byte)segmentId); + } + } + /// /// Allocates the loop-restoration unit grid described by the active frame header. /// @@ -440,8 +551,10 @@ internal partial class Av1FrameInfo /// The containing superblock. public void UpdateModeInfo(Av1BlockModeInfo modeInfo, Av1SuperblockInfo superblockInfo) { + Point modeInfoPosition = this.GetModeInfoPosition(superblockInfo.Position, modeInfo.PositionInSuperblock); this.modeInfos[this.modeInfoMap.NextIndex] = modeInfo; - this.modeInfoMap.Update(this.GetModeInfoPosition(superblockInfo.Position, modeInfo.PositionInSuperblock), modeInfo.BlockSize); + this.UpdateRetainedMotionField(modeInfo, modeInfoPosition); + this.modeInfoMap.Update(modeInfoPosition, modeInfo.BlockSize); } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1InterIntraMode.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1InterIntraMode.cs new file mode 100644 index 000000000..bd101ab63 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1InterIntraMode.cs @@ -0,0 +1,30 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +/// +/// Identifies the intra predictor blended with a single-reference inter predictor. +/// +internal enum Av1InterIntraMode : byte +{ + /// + /// Uses a DC intra predictor. + /// + DC = 0, + + /// + /// Uses a vertical intra predictor. + /// + Vertical = 1, + + /// + /// Uses a horizontal intra predictor. + /// + Horizontal = 2, + + /// + /// Uses a smooth intra predictor. + /// + Smooth = 3, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MotionMode.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MotionMode.cs new file mode 100644 index 000000000..ed9977b1e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MotionMode.cs @@ -0,0 +1,25 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +/// +/// Identifies the motion model used to construct an AV1 inter predictor. +/// +internal enum Av1MotionMode : byte +{ + /// + /// Uses translational motion compensation without neighboring-block overlap. + /// + SimpleTranslation = 0, + + /// + /// Blends the block with predictions derived from overlapping above and left neighbors. + /// + Obmc = 1, + + /// + /// Uses a locally derived warped-motion model. + /// + Warped = 2, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs index f3707ef99..0f51e07aa 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs @@ -10,10 +10,30 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// /// Describes a decoded AV1 partition's block geometry, neighbors, and frame-boundary availability. /// -internal class Av1PartitionInfo +internal ref struct Av1PartitionInfo { /// - /// Initializes a new instance of the class. + /// The luma block width in samples. + /// + private int lumaWidthInPixels; + + /// + /// The shared chroma block width in samples. + /// + private int chromaWidthInPixels; + + /// + /// The luma block height in samples. + /// + private int lumaHeightInPixels; + + /// + /// The shared chroma block height in samples. + /// + private int chromaHeightInPixels; + + /// + /// Initializes a new instance of the structure. /// /// The decoded mode information for the partition block. /// The containing superblock. @@ -25,9 +45,6 @@ internal class Av1PartitionInfo this.SuperblockInfo = superblockInfo; this.IsChroma = isChroma; this.Type = partitionType; - this.ReferenceFrame = [-1, -1]; - this.WidthInPixels = new int[3]; - this.HeightInPixels = new int[3]; } /// @@ -101,9 +118,9 @@ internal class Av1PartitionInfo public Av1BlockModeInfo? LeftModeInfoForChroma { get; set; } /// - /// Gets or sets the reference-frame identifiers selected for the block. + /// Gets the reference-frame types selected for the block. /// - public int[] ReferenceFrame { get; set; } + public Span ReferenceFrames => this.ModeInfo.ReferenceFrames; /// /// Gets the signed distance from the block to the left frame edge in one-eighth-sample units. @@ -126,19 +143,23 @@ internal class Av1PartitionInfo public int ModeBlockToBottomEdge { get; private set; } /// - /// Gets the block width in samples for each color plane. + /// Gets or sets the neighboring luma samples used by chroma-from-luma prediction. /// - public int[] WidthInPixels { get; private set; } + public Av1ChromaFromLumaContext? ChromaFromLumaContext { get; internal set; } /// - /// Gets the block height in samples for each color plane. + /// Gets the block width in samples for a color plane. /// - public int[] HeightInPixels { get; private set; } + /// The luma, blue-difference, or red-difference plane. + /// The block width in samples for the requested plane. + public int GetWidthInPixels(Av1Plane plane) => plane == Av1Plane.Y ? this.lumaWidthInPixels : this.chromaWidthInPixels; /// - /// Gets or sets the neighboring luma samples used by chroma-from-luma prediction. + /// Gets the block height in samples for a color plane. /// - public Av1ChromaFromLumaContext? ChromaFromLumaContext { get; internal set; } + /// The luma, blue-difference, or red-difference plane. + /// The block height in samples for the requested plane. + public int GetHeightInPixels(Av1Plane plane) => plane == Av1Plane.Y ? this.lumaHeightInPixels : this.chromaHeightInPixels; /// /// Computes tile-neighbor availability, frame-edge distances, and per-plane block dimensions. @@ -167,14 +188,10 @@ internal class Av1PartitionInfo // The bitstream expresses block size on the luma grid. Chroma dimensions are derived by // subsampling that grid while retaining at least one 4x4 chroma unit for narrow blocks. const int modeInfoSize = 1 << Av1Constants.ModeInfoSizeLog2; - this.WidthInPixels[0] = bw4 * modeInfoSize; - this.HeightInPixels[0] = bh4 * modeInfoSize; - - this.WidthInPixels[1] = Math.Max(1, bw4 >> subX) * modeInfoSize; - this.HeightInPixels[1] = Math.Max(1, bh4 >> subY) * modeInfoSize; - - this.WidthInPixels[2] = Math.Max(1, bw4 >> subX) * modeInfoSize; - this.HeightInPixels[2] = Math.Max(1, bh4 >> subY) * modeInfoSize; + this.lumaWidthInPixels = bw4 * modeInfoSize; + this.lumaHeightInPixels = bh4 * modeInfoSize; + this.chromaWidthInPixels = Math.Max(1, bw4 >> subX) * modeInfoSize; + this.chromaHeightInPixels = Math.Max(1, bh4 >> subY) * modeInfoSize; } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1ReferenceFrameType.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1ReferenceFrameType.cs new file mode 100644 index 000000000..ffc1c98af --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1ReferenceFrameType.cs @@ -0,0 +1,55 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +/// +/// Identifies the current or retained frame used to predict an AV1 coding block. +/// +internal enum Av1ReferenceFrameType : sbyte +{ + /// + /// Indicates that the optional secondary reference is absent. + /// + None = -1, + + /// + /// References the current frame for intra prediction. + /// + Intra = 0, + + /// + /// References the most recent forward prediction frame. + /// + Last = 1, + + /// + /// References the second most recent forward prediction frame. + /// + Last2 = 2, + + /// + /// References the third most recent forward prediction frame. + /// + Last3 = 3, + + /// + /// References the long-term golden forward prediction frame. + /// + Golden = 4, + + /// + /// References the nearest backward prediction frame. + /// + Backward = 5, + + /// + /// References the secondary alternate backward prediction frame. + /// + Alternate2 = 6, + + /// + /// References the alternate backward prediction frame. + /// + Alternate = 7, +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs index 3d43797ee..d640d2a22 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs @@ -8,6 +8,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -17,40 +18,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// internal sealed class Av1TileReader : IAv1TileReader, IDisposable { - /// - /// The default self-guided restoration projection coefficients for each color plane. - /// - private static readonly int[] SgrprojXqdMid = [-32, 31]; - - /// - /// The default Wiener restoration taps retained between restoration units. - /// - private static readonly int[] WienerTapsMid = [3, -7, 15]; - - /// - /// The minimum transmitted value for each independent Wiener coefficient. - /// - private static readonly int[] WienerCoefficientMinimum = [-5, -23, -17]; - - /// - /// The number of possible transmitted values for each independent Wiener coefficient. - /// - private static readonly int[] WienerCoefficientValueCount = [16, 32, 64]; - - /// - /// The subexponential group-size exponent for each independent Wiener coefficient. - /// - private static readonly int[] WienerCoefficientSubexponentialK = [1, 2, 3]; - - /// - /// The two self-guided filter radii selected by each parameter-set index. - /// - private static readonly int[][] SgrProjectionRadii = - [ - [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], - [2, 1], [2, 1], [0, 1], [0, 1], [0, 1], [0, 1], [2, 0], [2, 0] - ]; - /// /// The minimum value of the first self-guided projection coefficient. /// @@ -72,31 +39,25 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable private const int SgrProjectionSubexponentialK = 4; /// - /// Maps packed coefficient sign classes to their signed contribution to the DC context. - /// - private static readonly int[] Signs = [0, -1, 1]; - - /// - /// Maps the summed neighboring DC signs to the AV1 DC-sign entropy context. - /// - private static readonly int[] DcSignContexts = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]; - - /// - /// Maps the weighted palette-neighbor score hash to its color-index entropy context. + /// The two self-guided filter radii selected by each parameter-set index. /// - private static readonly int[] PaletteColorIndexContexts = [-1, -1, 0, -1, -1, 4, 3, 2, 1]; + private static readonly int[][] SgrProjectionRadii = + [ + [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], [2, 1], + [2, 1], [2, 1], [0, 1], [0, 1], [0, 1], [0, 1], [2, 0], [2, 0] + ]; /// - /// Stores the preceding self-guided restoration coefficients for each color plane. + /// Stores two preceding self-guided restoration coefficients for each of the three color planes, indexed by + /// (plane * 2) + coefficient. /// - private int[][] referenceSgrXqd = []; + private InlineArray6 referenceSgrXqd; /// - /// Stores the preceding vertical and horizontal Wiener taps for each color plane. + /// Stores three preceding Wiener taps for both passes of each of the three color planes, indexed by + /// (((plane * 2) + pass) * 3) + tap. /// - private int[][][] referenceLrWiener = []; + private InlineArray18 referenceLrWiener; /// /// Tracks entropy, partition, and transform state above the current block. @@ -118,11 +79,6 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// private readonly int[] currentDeltaLoopFilter = new int[Av1Constants.FrameLoopFilterCount]; - /// - /// Stores the segment identifier covering each 4x4 frame position. - /// - private readonly int[][] segmentIds = []; - /// /// Stores per-plane transform counts for each forced 64x64 residual region. /// @@ -158,6 +114,16 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// private readonly IAv1FrameDecoder? frameDecoder; + /// + /// The decoder-session entropy contexts reused by every tile in the current frame. + /// + private readonly Av1FrameEntropyContexts entropyContexts; + + /// + /// The retained primary frame whose segment map supplies temporal segment-ID predictions. + /// + private readonly Av1FrameInfo? primaryReferenceFrameInfo; + /// /// Initializes a new instance of the class for syntax parsing without reconstruction. /// @@ -165,20 +131,57 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The active AV1 sequence header. /// The frame header whose tiles will be parsed. public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + : this(configuration, sequenceHeader, frameHeader, new(frameHeader.QuantizationParameters.BaseQIndex), null, null) + { + } + + /// + /// Initializes a new instance of the class with decoder-session entropy state. + /// + /// The decoder configuration. + /// The active AV1 sequence header. + /// The frame header whose tiles will be parsed. + /// The entropy contexts reused by the owning decoder session. + /// + /// The retained primary-reference entropy context, or when the frame selects defaults. + /// + /// + /// The retained reconstructed frames, or for the intra-only syntax reader. + /// + public Av1TileReader( + Configuration configuration, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1FrameEntropyContexts entropyContexts, + Av1FrameEntropyContext? primaryReferenceContext, + Av1ReferenceFrameStore? referenceFrames) { this.FrameHeader = frameHeader; this.configuration = configuration; this.SequenceHeader = sequenceHeader; + this.entropyContexts = entropyContexts; + this.entropyContexts.BeginFrame(frameHeader.QuantizationParameters.BaseQIndex, primaryReferenceContext); // FrameInfo owns all traversal-order records and coefficient storage produced by the tile readers. this.FrameInfo = new(this.SequenceHeader); - this.FrameInfo.InitializeLoopRestoration(this.SequenceHeader, this.FrameHeader); - this.segmentIds = new int[this.FrameHeader.ModeInfoRowCount][]; - for (int y = 0; y < this.FrameHeader.ModeInfoRowCount; y++) + if (referenceFrames is not null) { - this.segmentIds[y] = new int[this.FrameHeader.ModeInfoColumnCount]; + // Only the production decoder owns reconstructed references. Header-only intra readers retain their + // existing allocation profile and cannot reach inter mode parsing. + this.FrameInfo.InitializeMotionField(this.SequenceHeader, this.FrameHeader, referenceFrames); + + byte? primaryReferenceSlot = this.FrameHeader.PrimaryReferenceSlot; + if (primaryReferenceSlot is not null) + { + // The uncompressed-header parser has already validated this slot. Keep only its frame-state owner; + // segment samples remain in the retained frame and are copied only for whole-map inheritance. + this.primaryReferenceFrameInfo = referenceFrames.Resolve(primaryReferenceSlot.Value)!.FrameInfo; + } } + this.FrameInfo.InitializeSegmentIds(this.FrameHeader, this.primaryReferenceFrameInfo); + this.FrameInfo.InitializeLoopRestoration(this.SequenceHeader, this.FrameHeader); + // Above contexts span the aligned frame width, while left contexts are reused for each superblock row. int planesCount = sequenceHeader.ColorConfig.PlaneCount; int superblockColumnCount = @@ -216,6 +219,48 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable : this(configuration, sequenceHeader, frameHeader) => this.frameDecoder = frameDecoder; + /// + /// Gets the default self-guided restoration projection coefficients for each color plane. + /// + private static ReadOnlySpan SgrprojXqdMid => [-32, 31]; + + /// + /// Gets the default Wiener restoration taps retained between restoration units. + /// + private static ReadOnlySpan WienerTapsMid => [3, -7, 15]; + + /// + /// Gets the minimum transmitted value for each independent Wiener coefficient. + /// + private static ReadOnlySpan WienerCoefficientMinimum => [-5, -23, -17]; + + /// + /// Gets the number of possible transmitted values for each independent Wiener coefficient. + /// + private static ReadOnlySpan WienerCoefficientValueCount => [16, 32, 64]; + + /// + /// Gets the subexponential group-size exponent for each independent Wiener coefficient. + /// + private static ReadOnlySpan WienerCoefficientSubexponentialK => [1, 2, 3]; + + /// + /// Gets the signed DC-context contribution for each packed coefficient sign class. + /// + private static ReadOnlySpan Signs => [0, -1, 1]; + + /// + /// Gets the AV1 DC-sign entropy context for each summed neighboring sign value. + /// + private static ReadOnlySpan DcSignContexts => [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]; + + /// + /// Gets the color-index entropy context for each weighted palette-neighbor score hash. + /// + private static ReadOnlySpan PaletteColorIndexContexts => [-1, -1, 0, -1, -1, 4, 3, 2, 1]; + /// /// Gets the frame header whose tile syntax is being parsed. /// @@ -227,10 +272,20 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable public ObuSequenceHeader SequenceHeader { get; } /// - /// Gets the frame-owned mode, transform, coefficient, quantizer, and filter state populated by tile parsing. + /// Gets the frame-owned mode, motion, transform, coefficient, quantizer, and filter state populated by tile parsing. /// public Av1FrameInfo FrameInfo { get; } + /// + /// Gets the completed frame entropy context selected by the context-update tile. + /// + /// + /// The context contains either normative defaults or the selected primary-reference state until the signaled + /// update tile has decoded successfully. Callers that retain it beyond this reader's frame lifecycle must copy it + /// through . + /// + public Av1FrameEntropyContext FrameEntropyContext => this.entropyContexts.Published; + /// /// Returns the tile-neighbor context storage to the configured memory allocator. /// @@ -248,12 +303,17 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// Corresponds to parse_tile in SVT-AV1. public void ReadTile(Span tileData, int tileNum) { + // AV1 tiles never inherit adaptation from another tile in the same frame. Reusing one graph is safe because + // parsing is sequential and every entry is restored from the unchanged frame base before the range decoder is + // constructed. + this.entropyContexts.Working.CopyFrom(this.entropyContexts.Base); + // The frame syntax exposes a disable flag, while the range reader follows libaom's positive // allow_update_cdf convention. Av1SymbolDecoder reader = new( this.configuration, tileData, - this.FrameHeader.QuantizationParameters.BaseQIndex, + this.entropyContexts.Working, !this.FrameHeader.DisableCdfUpdate); int tileColumnIndex = tileNum % this.FrameHeader.TilesInfo.TileColumnCount; @@ -268,18 +328,18 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable this.ClearLoopFilterDelta(); int planesCount = this.SequenceHeader.ColorConfig.PlaneCount; - // Restoration coefficients are differentially coded, so each tile begins from the AV1 defaults. - this.referenceSgrXqd = new int[planesCount][]; - this.referenceLrWiener = new int[planesCount][][]; + // AV1 fixes restoration reference storage at three planes, two directions or projection coefficients, and + // three transmitted Wiener taps. Populate the inline value storage in place so every tile starts from the + // normative differential-coding defaults without constructing jagged arrays. + Span sgrReferences = this.referenceSgrXqd; + Span wienerReferences = this.referenceLrWiener; for (int plane = 0; plane < planesCount; plane++) { - this.referenceSgrXqd[plane] = new int[2]; - Array.Copy(SgrprojXqdMid, this.referenceSgrXqd[plane], SgrprojXqdMid.Length); - this.referenceLrWiener[plane] = new int[2][]; + SgrprojXqdMid.CopyTo(sgrReferences.Slice(plane * 2, 2)); for (int pass = 0; pass < 2; pass++) { - this.referenceLrWiener[plane][pass] = new int[Av1Constants.WienerCoefficientCount]; - Array.Copy(WienerTapsMid, this.referenceLrWiener[plane][pass], WienerTapsMid.Length); + int referenceOffset = ((plane * 2) + pass) * Av1Constants.WienerCoefficientCount; + WienerTapsMid.CopyTo(wienerReferences.Slice(referenceOffset, Av1Constants.WienerCoefficientCount)); } } @@ -309,6 +369,29 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable this.frameDecoder?.DecodeSuperblock(modeInfoPosition, superblockInfo, tileInfo); } } + + // Range decoding may read implicit zero padding while normalizing its final interval. Validate the logical + // stopping position before publishing either pixels or adapted CDF state so a truncated tile cannot commit. + reader.ValidateTrailingBits(); + + if (!this.FrameHeader.DisableFrameEndUpdateCdf && tileNum == this.FrameHeader.TilesInfo.ContextUpdateTileId) + { + // libaom publishes only context_update_tile_id after every tile has independently started from the frame + // base, then clears its CDF counters. Snapshotting into a third reusable graph preserves the unchanged base + // for tiles that follow the selected tile in bitstream order. + this.entropyContexts.Working.SnapshotTo(this.entropyContexts.Published); + } + } + + /// + /// Completes the current coded frame. + /// + public void CompleteFrame() + { + // Tile parsing and optional incremental superblock reconstruction finish inside ReadTile. The owning AV1 + // decoder uses this lifecycle boundary to assemble native planes. A directly created tile reader owns its + // neighbor-context rents, so the same boundary must return them before ObuReader releases the frame instance. + this.Dispose(); } /// @@ -425,15 +508,16 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable destination[0] = 0; for (int coefficient = firstCoefficient; coefficient < Av1Constants.WienerCoefficientCount; coefficient++) { + int referenceIndex = (((plane * 2) + pass) * Av1Constants.WienerCoefficientCount) + coefficient; int minimum = WienerCoefficientMinimum[coefficient]; int value = reader.ReadReferenceSubexponential( WienerCoefficientValueCount[coefficient], WienerCoefficientSubexponentialK[coefficient], - this.referenceLrWiener[plane][pass][coefficient] - minimum); + this.referenceLrWiener[referenceIndex] - minimum); value += minimum; destination[coefficient] = value; - this.referenceLrWiener[plane][pass][coefficient] = value; + this.referenceLrWiener[referenceIndex] = value; } } } @@ -449,7 +533,8 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable unit.SgrParameterSet = reader.ReadLiteral(4); int[] radii = SgrProjectionRadii[unit.SgrParameterSet]; int[] coefficients = unit.SgrProjectionCoefficients; - int[] references = this.referenceSgrXqd[plane]; + Span allReferences = this.referenceSgrXqd; + Span references = allReferences.Slice(plane * 2, 2); if (radii[0] == 0) { coefficients[0] = 0; @@ -472,7 +557,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable coefficients[1] = ReadSgrProjectionCoefficient(ref reader, references[1], SgrProjectionCoefficient1Minimum); } - coefficients.CopyTo(references, 0); + coefficients.CopyTo(references); } /// @@ -681,15 +766,15 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable partitionInfo.PopulateModeInfoNeighbors(this.SequenceHeader.ColorConfig); - this.ReadModeInfo(ref reader, partitionInfo, tileInfo); - this.ReadPaletteTokens(ref reader, partitionInfo); - this.ReadBlockTransformSize(ref reader, modeInfoLocation, partitionInfo, superblockInfo, tileInfo); + this.ReadModeInfo(ref reader, ref partitionInfo, tileInfo); + this.ReadPaletteTokens(ref reader, ref partitionInfo); + this.ReadBlockTransformSize(ref reader, modeInfoLocation, ref partitionInfo, superblockInfo, tileInfo); if (partitionInfo.ModeInfo.Skip) { - this.ResetSkipContext(partitionInfo, tileInfo); + this.ResetSkipContext(ref partitionInfo, tileInfo); } - this.Residual(ref reader, partitionInfo, superblockInfo, tileInfo, blockSize); + this.Residual(ref reader, ref partitionInfo, superblockInfo, tileInfo, blockSize); // Store the record only after all syntax has populated it, then map every covered 4x4 position. this.FrameInfo.UpdateModeInfo(blockModeInfo, superblockInfo); @@ -701,7 +786,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The skipped block and its frame position. /// The active tile boundaries. /// Corresponds to reset_skip_context in SVT-AV1. - private void ResetSkipContext(Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo) + private void ResetSkipContext(ref Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo) { int planesCount = this.SequenceHeader.ColorConfig.PlaneCount; for (int i = 0; i < planesCount; i++) @@ -728,7 +813,12 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The active tile boundaries. /// The coding block size. /// Implements AV1 section 5.11.34 and corresponds to parse_residual in SVT-AV1. - private void Residual(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo, Av1BlockSize blockSize) + private void Residual( + ref Av1SymbolDecoder reader, + ref Av1PartitionInfo partitionInfo, + Av1SuperblockInfo superblockInfo, + Av1TileInfo tileInfo, + Av1BlockSize blockSize) { int maxBlocksWide = partitionInfo.GetMaxBlockWide(blockSize, false); int maxBlocksHigh = partitionInfo.GetMaxBlockHigh(blockSize, false); @@ -823,7 +913,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable Span coefficientBuffer = superblockInfo.GetCoefficients((Av1Plane)plane)[coefficientIndex..]; endOfBlock = this.ParseTransformBlock( ref reader, - partitionInfo, + ref partitionInfo, tileInfo, coefficientBuffer, transformInfo, @@ -913,7 +1003,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// private int ParseTransformBlock( ref Av1SymbolDecoder reader, - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo, Span coefficientBuffer, Av1TransformInfo transformInfo, @@ -960,7 +1050,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable endOfBlock = this.ParseCoefficients( ref reader, - partitionInfo, + ref partitionInfo, blockRow, blockColumn, aboveContextOffset, @@ -992,7 +1082,18 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// /// Implements AV1 section 5.11.39 using the traversal shape of the corresponding SVT-AV1 implementation. /// - private int ParseCoefficients(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo, int blockRow, int blockColumn, int aboveOffset, int leftOffset, int plane, Av1TransformBlockContext transformBlockContext, Av1TransformSize transformSize, Av1TransformInfo transformInfo, Span coefficientBuffer) + private int ParseCoefficients( + ref Av1SymbolDecoder reader, + ref Av1PartitionInfo partitionInfo, + int blockRow, + int blockColumn, + int aboveOffset, + int leftOffset, + int plane, + Av1TransformBlockContext transformBlockContext, + Av1TransformSize transformSize, + Av1TransformInfo transformInfo, + Span coefficientBuffer) { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -1063,7 +1164,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable { uint sign = (uint)aboveContext[k] >> Av1Constants.CoefficientContextBitCount; DebugGuard.MustBeLessThanOrEqualTo(sign, 2U, nameof(sign)); - dcSign += Signs[sign]; + dcSign += Signs[(int)sign]; } while (++k < transformBlockUnitWideCount); @@ -1072,7 +1173,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable { uint sign = (uint)leftContext[k] >> Av1Constants.CoefficientContextBitCount; DebugGuard.MustBeLessThanOrEqualTo(sign, 2U, nameof(sign)); - dcSign += Signs[sign]; + dcSign += Signs[(int)sign]; } while (++k < transformBlockUnitHighCount); @@ -1168,7 +1269,12 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// A value indicating whether transform-size selection syntax is allowed at this node. /// The selected transform size. /// Implements AV1 section 5.11.15. - private Av1TransformSize ReadTransformSize(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo, bool allowSelect) + private Av1TransformSize ReadTransformSize( + ref Av1SymbolDecoder reader, + ref Av1PartitionInfo partitionInfo, + Av1SuperblockInfo superblockInfo, + Av1TileInfo tileInfo, + bool allowSelect) { Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; if (this.FrameHeader.LosslessArray[modeInfo.SegmentId]) @@ -1178,7 +1284,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable if (modeInfo.BlockSize > Av1BlockSize.Block4x4 && allowSelect && this.FrameHeader.TransformMode == Av1TransformMode.Select) { - return this.ReadSelectedTransformSize(ref reader, partitionInfo, superblockInfo, tileInfo); + return this.ReadSelectedTransformSize(ref reader, ref partitionInfo, superblockInfo, tileInfo); } return modeInfo.BlockSize.GetMaximumTransformSize(); @@ -1192,7 +1298,11 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The containing superblock. /// The active tile boundaries. /// The decoded transform size. - private Av1TransformSize ReadSelectedTransformSize(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo) + private Av1TransformSize ReadSelectedTransformSize( + ref Av1SymbolDecoder reader, + ref Av1PartitionInfo partitionInfo, + Av1SuperblockInfo superblockInfo, + Av1TileInfo tileInfo) { int context = 0; Av1TransformSize maxTransformSize = partitionInfo.ModeInfo.BlockSize.GetMaximumTransformSize(); @@ -1232,17 +1342,22 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The containing superblock. /// The active tile boundaries. /// Implements AV1 section 5.11.16 and corresponds to read_block_tx_size in SVT-AV1. - private void ReadBlockTransformSize(ref Av1SymbolDecoder reader, Point modeInfoLocation, Av1PartitionInfo partitionInfo, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo) + private void ReadBlockTransformSize( + ref Av1SymbolDecoder reader, + Point modeInfoLocation, + ref Av1PartitionInfo partitionInfo, + Av1SuperblockInfo superblockInfo, + Av1TileInfo tileInfo) { Av1BlockSize blockSize = partitionInfo.ModeInfo.BlockSize; int block4x4Width = blockSize.Get4x4WideCount(); int block4x4Height = blockSize.Get4x4HighCount(); // HEIF still-image decoding follows the independently decodable intra-frame transform-size branch. - Av1TransformSize transformSize = this.ReadTransformSize(ref reader, partitionInfo, superblockInfo, tileInfo, true); + Av1TransformSize transformSize = this.ReadTransformSize(ref reader, ref partitionInfo, superblockInfo, tileInfo, true); this.aboveNeighborContext.UpdateTransformation(modeInfoLocation, tileInfo, transformSize, blockSize, false); this.leftNeighborContext.UpdateTransformation(modeInfoLocation, superblockInfo, transformSize, blockSize, false); - this.UpdateTransformInfo(partitionInfo, superblockInfo, blockSize, transformSize); + this.UpdateTransformInfo(ref partitionInfo, superblockInfo, blockSize, transformSize); } /// @@ -1252,7 +1367,11 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The containing superblock and transform storage. /// The coding block size. /// The selected luma transform size. - private unsafe void UpdateTransformInfo(Av1PartitionInfo partitionInfo, Av1SuperblockInfo superblockInfo, Av1BlockSize blockSize, Av1TransformSize transformSize) + private unsafe void UpdateTransformInfo( + ref Av1PartitionInfo partitionInfo, + Av1SuperblockInfo superblockInfo, + Av1BlockSize blockSize, + Av1TransformSize transformSize) { int transformInfoYIndex = partitionInfo.ModeInfo.GetFirstTransformLocation(Av1PlaneType.Y); int transformInfoUvIndex = partitionInfo.ModeInfo.GetFirstTransformLocation(Av1PlaneType.Uv); @@ -1367,13 +1486,13 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block. /// Implements AV1 section 5.11.49. - private void ReadPaletteTokens(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void ReadPaletteTokens(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; if (modeInfo.GetPaletteSize(Av1PlaneType.Y) != 0) { GetPaletteMapDimensions( - partitionInfo, + ref partitionInfo, Av1PlaneType.Y, this.SequenceHeader.ColorConfig, out int planeWidth, @@ -1396,7 +1515,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable if (modeInfo.GetPaletteSize(Av1PlaneType.Uv) != 0) { GetPaletteMapDimensions( - partitionInfo, + ref partitionInfo, Av1PlaneType.Uv, this.SequenceHeader.ColorConfig, out int planeWidth, @@ -1424,10 +1543,10 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The current coding block. /// The active tile boundaries. /// Implements the intra-frame branch of AV1 section 5.11.6. - private void ReadModeInfo(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo) + private void ReadModeInfo(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo) { DebugGuard.IsTrue(this.FrameHeader.FrameType is ObuFrameType.KeyFrame or ObuFrameType.IntraOnlyFrame, "Only INTRA frames supported."); - this.ReadIntraFrameModeInfo(ref reader, partitionInfo, tileInfo); + this.ReadIntraFrameModeInfo(ref reader, ref partitionInfo, tileInfo); } /// @@ -1437,30 +1556,30 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The current coding block and its neighbors. /// The active tile boundaries. /// Implements AV1 section 5.11.7. - private void ReadIntraFrameModeInfo(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo) + private void ReadIntraFrameModeInfo(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo, Av1TileInfo tileInfo) { if (this.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip) { - this.IntraSegmentId(ref reader, partitionInfo); + this.IntraSegmentId(ref reader, ref partitionInfo); } - partitionInfo.ModeInfo.Skip = this.ReadSkip(ref reader, partitionInfo); + partitionInfo.ModeInfo.Skip = this.ReadSkip(ref reader, ref partitionInfo); if (!this.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip) { - this.IntraSegmentId(ref reader, partitionInfo); + this.IntraSegmentId(ref reader, ref partitionInfo); } - this.ReadCdef(ref reader, partitionInfo); + this.ReadCdef(ref reader, ref partitionInfo); if (this.FrameHeader.DeltaQParameters.IsPresent) { - this.ReadDeltaQuantizerIndex(ref reader, partitionInfo); - this.ReadDeltaLoopFilter(ref reader, partitionInfo); + this.ReadDeltaQuantizerIndex(ref reader, ref partitionInfo); + this.ReadDeltaLoopFilter(ref reader, ref partitionInfo); } // Independently decodable still-image blocks reference only the current intra frame. - partitionInfo.ReferenceFrame[0] = 0; - partitionInfo.ReferenceFrame[1] = -1; + partitionInfo.ReferenceFrames[0] = Av1ReferenceFrameType.Intra; + partitionInfo.ReferenceFrames[1] = Av1ReferenceFrameType.None; partitionInfo.ModeInfo.SetPaletteSizes(0, 0); bool useIntraBlockCopy = false; if (this.AllowIntraBlockCopy()) @@ -1472,17 +1591,17 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable { partitionInfo.ModeInfo.UseIntraBlockCopy = true; partitionInfo.ModeInfo.YMode = Av1PredictionMode.DC; - partitionInfo.ModeInfo.UvMode = Av1PredictionMode.DC; + partitionInfo.ModeInfo.UvMode = Av1ChromaPredictionMode.DC; Av1MotionVector reference = Av1IntraBlockCopy.FindReference( - partitionInfo, + ref partitionInfo, tileInfo, this.SequenceHeader.SuperblockModeInfoSize, this.displacementVectorCandidates, this.displacementVectorWeights); Av1MotionVector displacement = reader.ReadDisplacementVector(reference); - if (!Av1IntraBlockCopy.IsValid(displacement, partitionInfo, tileInfo, this.SequenceHeader)) + if (!Av1IntraBlockCopy.IsValid(displacement, ref partitionInfo, tileInfo, this.SequenceHeader)) { throw new InvalidImageContentException("Invalid AV1 intra-block-copy displacement vector."); } @@ -1499,19 +1618,22 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable if (partitionInfo.IsChroma && !this.SequenceHeader.ColorConfig.IsMonochrome) { - partitionInfo.ModeInfo.UvMode = reader.ReadIntraModeUv(partitionInfo.ModeInfo.YMode, this.IsChromaForLumaAllowed(partitionInfo)); - if (partitionInfo.ModeInfo.UvMode == Av1PredictionMode.UvChromaFromLuma) + partitionInfo.ModeInfo.UvMode = reader.ReadIntraModeUv( + partitionInfo.ModeInfo.YMode, + this.IsChromaForLumaAllowed(ref partitionInfo)); + + if (partitionInfo.ModeInfo.UvMode == Av1ChromaPredictionMode.ChromaFromLuma) { ReadChromaFromLumaAlphas(ref reader, partitionInfo.ModeInfo); } partitionInfo.ModeInfo.SetAngleDelta( Av1PlaneType.Uv, - IntraAngleInfo(ref reader, partitionInfo.ModeInfo.UvMode, partitionInfo.ModeInfo.BlockSize)); + IntraAngleInfo(ref reader, partitionInfo.ModeInfo.UvMode.ToLumaMode(), partitionInfo.ModeInfo.BlockSize)); } else { - partitionInfo.ModeInfo.UvMode = Av1PredictionMode.DC; + partitionInfo.ModeInfo.UvMode = Av1ChromaPredictionMode.DC; } if (partitionInfo.ModeInfo.BlockSize >= Av1BlockSize.Block8x8 && @@ -1519,10 +1641,10 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable partitionInfo.ModeInfo.BlockSize.GetHeight() <= 64 && this.FrameHeader.AllowScreenContentTools) { - this.PaletteModeInfo(ref reader, partitionInfo); + this.PaletteModeInfo(ref reader, ref partitionInfo); } - this.FilterIntraModeInfo(ref reader, partitionInfo); + this.FilterIntraModeInfo(ref reader, ref partitionInfo); } } @@ -1540,7 +1662,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// /// The current coding block. /// when the lossless transform or block dimensions permit chroma-from-luma prediction; otherwise, . - private bool IsChromaForLumaAllowed(Av1PartitionInfo partitionInfo) + private bool IsChromaForLumaAllowed(ref Av1PartitionInfo partitionInfo) { if (this.FrameHeader.LosslessArray[partitionInfo.ModeInfo.SegmentId]) { @@ -1561,7 +1683,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// /// The tile symbol decoder. /// The current coding block. - private void FilterIntraModeInfo(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void FilterIntraModeInfo(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { partitionInfo.ModeInfo.UseFilterIntra = false; if (this.SequenceHeader.EnableFilterIntra && @@ -1584,7 +1706,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block. /// Implements AV1 section 5.11.46. - private void PaletteModeInfo(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void PaletteModeInfo(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; Av1BlockSize blockSize = modeInfo.BlockSize; @@ -1611,20 +1733,20 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable { yPaletteSize = reader.ReadPaletteSize(blockSizeContext, Av1PlaneType.Y); Span yColors = stackalloc ushort[Av1Constants.PaletteMaxSize]; - ReadPaletteColorsY(ref reader, partitionInfo, yPaletteSize, bitDepth, yColors); + ReadPaletteColorsY(ref reader, ref partitionInfo, yPaletteSize, bitDepth, yColors); modeInfo.SetPaletteColors(Av1Plane.Y, yColors[..yPaletteSize]); } } if (this.SequenceHeader.ColorConfig.PlaneCount > 1 && - modeInfo.UvMode == Av1PredictionMode.DC && + modeInfo.UvMode == Av1ChromaPredictionMode.DC && partitionInfo.IsChroma && reader.ReadPaletteUvMode(yPaletteSize != 0)) { uvPaletteSize = reader.ReadPaletteSize(blockSizeContext, Av1PlaneType.Uv); Span uColors = stackalloc ushort[Av1Constants.PaletteMaxSize]; Span vColors = stackalloc ushort[Av1Constants.PaletteMaxSize]; - ReadPaletteColorsUv(ref reader, partitionInfo, uvPaletteSize, bitDepth, uColors, vColors); + ReadPaletteColorsUv(ref reader, ref partitionInfo, uvPaletteSize, bitDepth, uColors, vColors); modeInfo.SetPaletteColors(Av1Plane.U, uColors[..uvPaletteSize]); modeInfo.SetPaletteColors(Av1Plane.V, vColors[..uvPaletteSize]); } @@ -1642,14 +1764,14 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The destination palette-color buffer. private static void ReadPaletteColorsY( ref Av1SymbolDecoder reader, - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, int paletteSize, int bitDepth, scoped Span colors) { Span colorCache = stackalloc ushort[Av1Constants.PaletteMaxSize * 2]; Span cachedColors = stackalloc ushort[Av1Constants.PaletteMaxSize]; - int cacheSize = GetPaletteCache(partitionInfo, Av1Plane.Y, colorCache); + int cacheSize = GetPaletteCache(ref partitionInfo, Av1Plane.Y, colorCache); int colorIndex = 0; for (int i = 0; i < cacheSize && colorIndex < paletteSize; i++) { @@ -1695,7 +1817,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The destination V palette-color buffer. private static void ReadPaletteColorsUv( ref Av1SymbolDecoder reader, - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, int paletteSize, int bitDepth, scoped Span uColors, @@ -1703,7 +1825,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable { Span colorCache = stackalloc ushort[Av1Constants.PaletteMaxSize * 2]; Span cachedColors = stackalloc ushort[Av1Constants.PaletteMaxSize]; - int cacheSize = GetPaletteCache(partitionInfo, Av1Plane.U, colorCache); + int cacheSize = GetPaletteCache(ref partitionInfo, Av1Plane.U, colorCache); int colorIndex = 0; for (int i = 0; i < cacheSize && colorIndex < paletteSize; i++) { @@ -1782,7 +1904,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The luma or U plane whose sorted base colors form the cache. /// The destination cache, which can hold both neighboring palettes. /// The number of colors written to . - private static int GetPaletteCache(Av1PartitionInfo partitionInfo, Av1Plane plane, Span cache) + private static int GetPaletteCache(ref Av1PartitionInfo partitionInfo, Av1Plane plane, scoped Span cache) { // AV1 deliberately excludes the block above at a 64-by-64 superblock-row boundary. int minimumSuperblockHeight = Av1BlockSize.Block64x64.Get4x4HighCount(); @@ -1886,7 +2008,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The number of plane-block rows inside the coded image. /// The number of plane-block columns inside the coded image. private static void GetPaletteMapDimensions( - Av1PartitionInfo partitionInfo, + ref Av1PartitionInfo partitionInfo, Av1PlaneType planeType, ObuColorConfig colorConfig, out int planeWidth, @@ -2103,28 +2225,17 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block. /// Implements AV1 section 5.11.8. - private void IntraSegmentId(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void IntraSegmentId(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { - if (this.FrameHeader.SegmentationParameters.Enabled) + if (!this.FrameHeader.SegmentationParameters.Enabled) { - this.ReadSegmentId(ref reader, partitionInfo); + // Disabled segmentation assigns the default ID without allocating or populating a retained map. + return; } - int blockWidth4x4 = partitionInfo.ModeInfo.BlockSize.Get4x4WideCount(); - int blockHeight4x4 = partitionInfo.ModeInfo.BlockSize.Get4x4HighCount(); - int modeInfoCountX = Math.Min(this.FrameHeader.ModeInfoColumnCount - partitionInfo.ColumnIndex, blockWidth4x4); - int modeInfoCountY = Math.Min(this.FrameHeader.ModeInfoRowCount - partitionInfo.RowIndex, blockHeight4x4); - int segmentId = partitionInfo.ModeInfo.SegmentId; - - // Later blocks predict from 4x4 positions, so replicate one block ID over its clipped frame coverage. - for (int y = 0; y < modeInfoCountY; y++) - { - int[] segmentRow = this.segmentIds[partitionInfo.RowIndex + y]; - for (int x = 0; x < modeInfoCountX; x++) - { - segmentRow[partitionInfo.ColumnIndex + x] = segmentId; - } - } + this.ReadSegmentId(ref reader, ref partitionInfo); + Point modeInfoPosition = new(partitionInfo.ColumnIndex, partitionInfo.RowIndex); + this.FrameInfo.SetSegmentId(partitionInfo.ModeInfo.BlockSize, modeInfoPosition, partitionInfo.ModeInfo.SegmentId); } /// @@ -2133,7 +2244,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block and its available neighbors. /// Implements AV1 section 5.11.9. - private void ReadSegmentId(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void ReadSegmentId(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { int predictor; int prevUL = -1; @@ -2143,17 +2254,17 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable int rowIndex = partitionInfo.RowIndex; if (partitionInfo.AvailableAbove && partitionInfo.AvailableLeft) { - prevUL = Av1SymbolContextHelper.GetSegmentId(this.segmentIds, rowIndex - 1, columnIndex - 1); + prevUL = this.FrameInfo.GetSegmentId(rowIndex - 1, columnIndex - 1); } if (partitionInfo.AvailableAbove) { - prevU = Av1SymbolContextHelper.GetSegmentId(this.segmentIds, rowIndex - 1, columnIndex); + prevU = this.FrameInfo.GetSegmentId(rowIndex - 1, columnIndex); } if (partitionInfo.AvailableLeft) { - prevL = Av1SymbolContextHelper.GetSegmentId(this.segmentIds, rowIndex, columnIndex - 1); + prevL = this.FrameInfo.GetSegmentId(rowIndex, columnIndex - 1); } if (prevU == -1) @@ -2191,7 +2302,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block. /// Implements AV1 section 5.11.56 and corresponds to read_cdef in libaom. - private void ReadCdef(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void ReadCdef(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { if (partitionInfo.ModeInfo.Skip || this.FrameHeader.CodedLossless || !this.SequenceHeader.EnableCdef || this.FrameHeader.AllowIntraBlockCopy) { @@ -2235,7 +2346,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// /// The tile symbol decoder. /// The current coding block and superblock delta storage. - private void ReadDeltaLoopFilter(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void ReadDeltaLoopFilter(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { if (!this.FrameHeader.DeltaLoopFilterParameters.IsPresent || partitionInfo.ModeInfo.PositionInSuperblock != Point.Empty) { @@ -2273,7 +2384,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block and its available neighbors. /// when the block omits residual coefficients; otherwise, . - private bool ReadSkip(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private bool ReadSkip(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { int segmentId = partitionInfo.ModeInfo.SegmentId; if (this.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip && @@ -2295,7 +2406,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// The tile symbol decoder. /// The current coding block and superblock quantizer storage. /// Corresponds to read_delta_qindex in SVT-AV1. - private void ReadDeltaQuantizerIndex(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) + private void ReadDeltaQuantizerIndex(ref Av1SymbolDecoder reader, ref Av1PartitionInfo partitionInfo) { if (!this.FrameHeader.DeltaQParameters.IsPresent || partitionInfo.ModeInfo.PositionInSuperblock != Point.Empty) { @@ -2416,4 +2527,17 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable } } } + + /// + /// Provides inline storage for the two self-guided restoration coefficients of each of the three AV1 planes. + /// + /// The stored value type. + [InlineArray(6)] + private struct InlineArray6 + { + /// + /// The first element in the compiler-expanded inline buffer. + /// + private T element; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs index 143b7ce37..4cbaeccae 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs @@ -485,7 +485,7 @@ internal partial class Av1TileWriter } Av1PredictionMode intra_luma_mode = macroBlockModeInfo.Block.Mode; - Av1PredictionMode intra_chroma_mode = macroBlockModeInfo.Block.UvMode; + Av1ChromaPredictionMode intra_chroma_mode = macroBlockModeInfo.Block.UvMode; if (IsIntraBlockCopyAllowed(pcs.Parent.FrameHeader/*, pcs.Parent.SliceType*/)) { WriteIntraBlockCopyInfo(ref writer, macroBlockModeInfo, blk_ptr); @@ -624,12 +624,12 @@ internal partial class Av1TileWriter Av1EncoderBlockStruct blk_ptr, Av1BlockSize blockSize, Av1PredictionMode lumaMode, - Av1PredictionMode chromaMode, + Av1ChromaPredictionMode chromaMode, bool isChromaFromLumaAllowed) { writer.WriteChromaMode(chromaMode, isChromaFromLumaAllowed, lumaMode); - if (chromaMode == Av1PredictionMode.UvChromaFromLuma) + if (chromaMode == Av1ChromaPredictionMode.ChromaFromLuma) { writer.WriteChromaFromLumaAlphas( blk_ptr.PredictionUnits[0].ChromaFromLumaIndex, @@ -638,7 +638,9 @@ internal partial class Av1TileWriter if (blockSize >= Av1BlockSize.Block8x8 && macroBlockModeInfo.Block.UvMode.IsDirectional()) { - writer.WriteAngleDelta(blk_ptr.PredictionUnits[0].AngleDelta[(int)Av1PlaneType.Uv] + Av1Constants.MaxAngleDelta, chromaMode); + writer.WriteAngleDelta( + blk_ptr.PredictionUnits[0].AngleDelta[(int)Av1PlaneType.Uv] + Av1Constants.MaxAngleDelta, + chromaMode.ToLumaMode()); } } @@ -712,7 +714,7 @@ internal partial class Av1TileWriter Point point) {/* Av1PredictionMode intra_luma_mode = macroBlockModeInfo.Mode; - Av1PredictionMode intra_chroma_mode = macroBlockModeInfo.ModeUv; + Av1ChromaPredictionMode intra_chroma_mode = macroBlockModeInfo.Block.UvMode; Av1PaletteModeInfo pmi = blk_ptr.PaletteInfo.pmi; int bsize_ctx = svt_aom_get_palette_bsize_ctx(bsize); @@ -729,7 +731,7 @@ internal partial class Av1TileWriter } } - bool uv_dc_pred = intra_chroma_mode == Av1PredictionMode.DC && is_chroma_reference(point, blockSize, 1, 1); + bool uv_dc_pred = intra_chroma_mode == Av1ChromaPredictionMode.DC && is_chroma_reference(point, blockSize, 1, 1); if (uv_dc_pred) { // assert(blk_ptr->palette_size[1] == 0); //remove when chroma is on diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs index 25570700b..1c59d01f7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs @@ -106,15 +106,34 @@ internal sealed class Av1BlockDecoder : IDisposable (this.sequenceHeader.ColorConfig.SubSamplingX ? ySize >> 2 : ySize) + (this.sequenceHeader.ColorConfig.SubSamplingY ? ySize >> 2 : ySize); - this.inverseQuantizationOwner = this.frameBuffer.MemoryAllocator.Allocate(inverseQuantizationSize); - this.transformWorkspaceOwner = this.frameBuffer.MemoryAllocator.Allocate(Av1TransformWorkspace.MaximumLength); - this.predictionScratchOwner = this.frameBuffer.MemoryAllocator.Allocate(Av1PredictionDecoder.ScratchLength); - this.predictionDecoder = new(sequenceHeader, frameHeader, this.predictionScratchOwner.Memory); - this.isLoopFilterEnabled = frameHeader.LoopFilterParameters.FilterLevel[0] != 0 || - frameHeader.LoopFilterParameters.FilterLevel[1] != 0; - - this.currentCoefficientIndex = new int[3]; - this.chromaFromLumaContext = new(sequenceHeader.ColorConfig); + IMemoryOwner? inverseQuantizationOwner = null; + IMemoryOwner? transformWorkspaceOwner = null; + IMemoryOwner? predictionScratchOwner = null; + try + { + inverseQuantizationOwner = this.frameBuffer.MemoryAllocator.Allocate(inverseQuantizationSize); + transformWorkspaceOwner = this.frameBuffer.MemoryAllocator.Allocate(Av1TransformWorkspace.MaximumLength); + predictionScratchOwner = this.frameBuffer.MemoryAllocator.Allocate(Av1PredictionDecoder.ScratchLength); + + this.inverseQuantizationOwner = inverseQuantizationOwner; + this.transformWorkspaceOwner = transformWorkspaceOwner; + this.predictionScratchOwner = predictionScratchOwner; + this.predictionDecoder = new(sequenceHeader, frameHeader, predictionScratchOwner.Memory); + this.isLoopFilterEnabled = frameHeader.LoopFilterParameters.FilterLevel[0] != 0 || + frameHeader.LoopFilterParameters.FilterLevel[1] != 0; + + this.currentCoefficientIndex = new int[3]; + this.chromaFromLumaContext = new(sequenceHeader.ColorConfig); + } + catch + { + // A constructor that does not return transfers no ownership to its caller. Unwind successful rents in + // reverse order so allocator diagnostics and pooled buffers remain balanced after any later allocation. + predictionScratchOwner?.Dispose(); + transformWorkspaceOwner?.Dispose(); + inverseQuantizationOwner?.Dispose(); + throw; + } } /// @@ -360,7 +379,7 @@ internal sealed class Av1BlockDecoder : IDisposable if (highBitDepth) { this.predictionDecoder.Decode( - partitionInfo, + ref partitionInfo, (Av1Plane)plane, transformSize, tileInfo, @@ -373,7 +392,7 @@ internal sealed class Av1BlockDecoder : IDisposable else { this.predictionDecoder.Decode( - partitionInfo, + ref partitionInfo, (Av1Plane)plane, transformSize, tileInfo, @@ -438,7 +457,7 @@ internal sealed class Av1BlockDecoder : IDisposable } // Store Luma for CFL if required! - if (plane == (int)Av1Plane.Y && StoreChromaFromLumaRequired(colorConfig, partitionInfo)) + if (plane == (int)Av1Plane.Y && StoreChromaFromLumaRequired(colorConfig, ref partitionInfo)) { // The predictor span begins on the previous row; CFL storage consumes reconstructed samples from // the transform block itself, hence the explicit one-stride advance for both sample pipelines. @@ -562,7 +581,7 @@ internal sealed class Av1BlockDecoder : IDisposable /// /// when chroma is present and the current luma block can contribute to a chroma-from-luma block. /// - private static bool StoreChromaFromLumaRequired(ObuColorConfig colorConfig, Av1PartitionInfo partitionInfo) + private static bool StoreChromaFromLumaRequired(ObuColorConfig colorConfig, ref Av1PartitionInfo partitionInfo) => !colorConfig.IsMonochrome && - (!partitionInfo.IsChroma || partitionInfo.ModeInfo.UvMode == Av1PredictionMode.UvChromaFromLuma); + (!partitionInfo.IsChroma || partitionInfo.ModeInfo.UvMode == Av1ChromaPredictionMode.ChromaFromLuma); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs new file mode 100644 index 000000000..10516e3d7 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixteen-point asymmetric discrete sine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator. + /// + internal readonly struct Adst16Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs new file mode 100644 index 000000000..f0cd6ff74 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the four-point asymmetric discrete sine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the four-point AV1 forward asymmetric discrete sine transform operator. + /// + internal readonly struct Adst4Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs new file mode 100644 index 000000000..0df4c4414 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the eight-point asymmetric discrete sine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the eight-point AV1 forward asymmetric discrete sine transform operator. + /// + internal readonly struct Adst8Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs new file mode 100644 index 000000000..3039ae930 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixteen-point discrete cosine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the sixteen-point AV1 forward discrete cosine transform operator. + /// + internal readonly struct Dct16Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs new file mode 100644 index 000000000..e662e6e08 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the thirty-two-point discrete cosine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the thirty-two-point AV1 forward discrete cosine transform operator. + /// + internal readonly struct Dct32Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs new file mode 100644 index 000000000..85eee7142 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the four-point discrete cosine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the four-point AV1 forward discrete cosine transform operator. + /// + internal readonly struct Dct4Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs new file mode 100644 index 000000000..8446f78a9 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixty-four-point discrete cosine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the sixty-four-point AV1 forward discrete cosine transform operator. + /// + internal readonly struct Dct64Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs new file mode 100644 index 000000000..c4c4acc7a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the eight-point discrete cosine forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the eight-point AV1 forward discrete cosine transform operator. + /// + internal readonly struct Dct8Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs new file mode 100644 index 000000000..61d85e1a9 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixteen-point identity forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the sixteen-point AV1 forward identity transform operator. + /// + internal readonly struct Identity16Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs new file mode 100644 index 000000000..7a65eaa01 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the thirty-two-point identity forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the thirty-two-point AV1 forward identity transform operator. + /// + internal readonly struct Identity32Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs new file mode 100644 index 000000000..f112053cf --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the four-point identity forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the four-point AV1 forward identity transform operator. + /// + internal readonly struct Identity4Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs new file mode 100644 index 000000000..24edb99e4 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs @@ -0,0 +1,29 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the eight-point identity forward transform operator. +/// +internal static partial class Av1ForwardTransformer +{ + /// + /// Defines the eight-point AV1 forward identity transform operator. + /// + internal readonly struct Identity8Operator : IAv1ForwardTransform1dOperator + { + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Operator.cs similarity index 91% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Operator.cs index 650d6a21b..f6b7ca07a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Operator.cs @@ -18,8 +18,38 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// Eight-bit blocks use saturating 16-bit stages where their normative ranges permit it; high-bit-depth and scalar /// fallback paths retain 32-bit stages. Both representations produce the same row-major coefficient contract. /// -internal static class Av1ForwardTransformer +internal static partial class Av1ForwardTransformer { + /// + /// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type. + /// + /// + /// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type + /// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch + /// inside the transform stages. + /// + internal interface IAv1ForwardTransform1dOperator + { + /// + /// Transforms the independent axes stored in each value lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct; + } + /// /// Resolves and applies the configured two-dimensional AV1 forward transform. /// @@ -65,40 +95,40 @@ internal static class Av1ForwardTransformer switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeColumn} column transform is not valid for {config.TransformSize}."); @@ -127,40 +157,40 @@ internal static class Av1ForwardTransformer switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeRow} row transform is not valid for {config.TransformSize}."); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst16Operator.cs new file mode 100644 index 000000000..c0b6ef9bd --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst16Operator.cs @@ -0,0 +1,574 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixteen-point asymmetric discrete sine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Adst16Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen spatial-domain residual values. + /// The sixteen-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[15]; + output[1] = input[0]; + output[2] = input[13]; + output[3] = input[2]; + output[4] = input[11]; + output[5] = input[4]; + output[6] = input[9]; + output[7] = input[6]; + output[8] = input[7]; + output[9] = input[8]; + output[10] = input[5]; + output[11] = input[10]; + output[12] = input[3]; + output[13] = input[12]; + output[14] = input[1]; + output[15] = input[14]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); + step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); + step[12] = output[12]; + step[13] = output[13]; + step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[8]; + output[2] = step[12]; + output[3] = -step[4]; + output[4] = step[6]; + output[5] = -step[14]; + output[6] = step[10]; + output[7] = -step[2]; + output[8] = step[3]; + output[9] = -step[11]; + output[10] = step[15]; + output[11] = -step[7]; + output[12] = step[5]; + output[13] = -step[13]; + output[14] = step[9]; + output[15] = -step[1]; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs new file mode 100644 index 000000000..ce6a60871 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs @@ -0,0 +1,147 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the four-point asymmetric discrete sine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the four-point AV1 inverse asymmetric discrete sine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Adst4Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform. + /// + /// The four frequency-domain coefficients. + /// The four spatial-domain residual values. + /// The stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the sine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + + // libaom widens the complete four-point factorization because the products retain their fixed-point scale + // until the final shift. The stage buffer is therefore unnecessary for this transform size. + long x0 = input[0]; + long x1 = input[1]; + long x2 = input[2]; + long x3 = input[3]; + + _ = step; + _ = stageRange; + + // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel. + if ((x0 | x1 | x2 | x3) == 0) + { + output[..4].Clear(); + return; + } + + // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3. + long s0 = sinpi[1] * x0; + long s1 = sinpi[2] * x0; + long s2 = sinpi[3] * x1; + long s3 = sinpi[4] * x2; + long s4 = sinpi[1] * x2; + long s5 = sinpi[2] * x3; + long s6 = sinpi[4] * x3; + long s7 = (x0 - x2) + x3; + + // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding. + s0 += s3; + s1 -= s4; + s3 = s2; + s2 = sinpi[3] * s7; + s0 += s5; + s1 -= s6; + x0 = s0 + s3; + x1 = s1 + s3; + x2 = s2; + x3 = (s0 + s1) - s3; + + output[0] = Av1Math.RoundShift(x0, cosBit); + output[1] = Av1Math.RoundShift(x1, cosBit); + output[2] = Av1Math.RoundShift(x2, cosBit); + output[3] = Av1Math.RoundShift(x3, cosBit); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit); + _ = step; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit); + _ = step; + _ = stageRange; + } + + /// + /// Applies the inverse four-point matrix to four independent axes. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The fixed-point precision of the sine constants. + private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector128 x0 = input.V0; + Vector128 x1 = input.V1; + Vector128 x2 = input.V2; + Vector128 x3 = input.V3; + + // The products retain the sine-table scale across the complete matrix. The bounded transform inputs make + // the optimized kernels' wrapping 32-bit multiply/add sequence valid until the terminal rounding shift. + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } + + /// + /// Applies the inverse four-point matrix to eight independent axes. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The fixed-point precision of the sine constants. + private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector256 x0 = input.V0; + Vector256 x1 = input.V1; + Vector256 x2 = input.V2; + Vector256 x3 = input.V3; + + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst8Operator.cs new file mode 100644 index 000000000..860bd8683 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst8Operator.cs @@ -0,0 +1,295 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the eight-point asymmetric discrete sine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Adst8Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform. + /// + /// The eight frequency-domain coefficients. + /// The eight spatial-domain residual values. + /// The eight-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[7]; + output[1] = input[0]; + output[2] = input[5]; + output[3] = input[2]; + output[4] = input[3]; + output[5] = input[4]; + output[6] = input[1]; + output[7] = input[6]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[4]; + output[2] = step[6]; + output[3] = -step[2]; + output[4] = step[3]; + output[5] = -step[7]; + output[6] = step[5]; + output[7] = -step[1]; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct16Operator.cs new file mode 100644 index 000000000..45fff6b79 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct16Operator.cs @@ -0,0 +1,481 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixteen-point discrete cosine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the 16-point AV1 inverse discrete cosine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Dct16Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative 16-point AV1 inverse discrete cosine transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen spatial-domain residual values. + /// The sixteen-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[8]; + output[2] = input[4]; + output[3] = input[12]; + output[4] = input[2]; + output[5] = input[10]; + output[6] = input[6]; + output[7] = input[14]; + output[8] = input[1]; + output[9] = input[9]; + output[10] = input[5]; + output[11] = input[13]; + output[12] = input[3]; + output[13] = input[11]; + output[14] = input[7]; + output[15] = input[15]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct32Operator.cs new file mode 100644 index 000000000..2b12ee021 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct32Operator.cs @@ -0,0 +1,1033 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the thirty-two-point discrete cosine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the 32-point AV1 inverse discrete cosine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Dct32Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative 32-point AV1 inverse discrete cosine transform. + /// + /// The 32 frequency-domain coefficients. + /// The 32 spatial-domain residual values. + /// The 32-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[16]; + output[2] = input[8]; + output[3] = input[24]; + output[4] = input[4]; + output[5] = input[20]; + output[6] = input[12]; + output[7] = input[28]; + output[8] = input[2]; + output[9] = input[18]; + output[10] = input[10]; + output[11] = input[26]; + output[12] = input[6]; + output[13] = input[22]; + output[14] = input[14]; + output[15] = input[30]; + output[16] = input[1]; + output[17] = input[17]; + output[18] = input[9]; + output[19] = input[25]; + output[20] = input[5]; + output[21] = input[21]; + output[22] = input[13]; + output[23] = input[29]; + output[24] = input[3]; + output[25] = input[19]; + output[26] = input[11]; + output[27] = input[27]; + output[28] = input[7]; + output[29] = input[23]; + output[30] = input[15]; + output[31] = input[31]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); + step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); + step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); + output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); + output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); + output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); + output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); + output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); + output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); + output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); + output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); + output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); + output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); + step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); + step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); + step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); + step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); + step[16] = output[16]; + step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); + step[19] = output[19]; + step[20] = output[20]; + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); + step[23] = output[23]; + step[24] = output[24]; + step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); + step[27] = output[27]; + step[28] = output[28]; + step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); + step[31] = output[31]; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + output[8] = step[8]; + output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); + output[11] = step[11]; + output[12] = step[12]; + output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); + output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); + output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); + output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); + output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); + output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); + output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); + step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); + step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); + step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); + step[30] = output[30]; + step[31] = output[31]; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + output[8] = step[8]; + output[9] = step[9]; + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); + output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); + output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); + output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); + output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); + step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); + step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); + step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); + output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); + output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); + output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct4Operator.cs new file mode 100644 index 000000000..9c8e2feaa --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct4Operator.cs @@ -0,0 +1,118 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the four-point discrete cosine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the four-point AV1 inverse discrete cosine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Dct4Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative four-point AV1 inverse discrete cosine transform. + /// + /// The four frequency-domain coefficients. + /// The four spatial-domain residual values. + /// The four-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output[0] = input[0]; + output[1] = input[2]; + output[2] = input[1]; + output[3] = input[3]; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct64Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct64Operator.cs new file mode 100644 index 000000000..e8ee0b417 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct64Operator.cs @@ -0,0 +1,2278 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixty-four-point discrete cosine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the 64-point AV1 inverse discrete cosine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Dct64Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative 64-point AV1 inverse discrete cosine transform. + /// + /// The 64 frequency-domain coefficients. + /// The 64 spatial-domain residual values. + /// The 64-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[32]; + output[2] = input[16]; + output[3] = input[48]; + output[4] = input[8]; + output[5] = input[40]; + output[6] = input[24]; + output[7] = input[56]; + output[8] = input[4]; + output[9] = input[36]; + output[10] = input[20]; + output[11] = input[52]; + output[12] = input[12]; + output[13] = input[44]; + output[14] = input[28]; + output[15] = input[60]; + output[16] = input[2]; + output[17] = input[34]; + output[18] = input[18]; + output[19] = input[50]; + output[20] = input[10]; + output[21] = input[42]; + output[22] = input[26]; + output[23] = input[58]; + output[24] = input[6]; + output[25] = input[38]; + output[26] = input[22]; + output[27] = input[54]; + output[28] = input[14]; + output[29] = input[46]; + output[30] = input[30]; + output[31] = input[62]; + output[32] = input[1]; + output[33] = input[33]; + output[34] = input[17]; + output[35] = input[49]; + output[36] = input[9]; + output[37] = input[41]; + output[38] = input[25]; + output[39] = input[57]; + output[40] = input[5]; + output[41] = input[37]; + output[42] = input[21]; + output[43] = input[53]; + output[44] = input[13]; + output[45] = input[45]; + output[46] = input[29]; + output[47] = input[61]; + output[48] = input[3]; + output[49] = input[35]; + output[50] = input[19]; + output[51] = input[51]; + output[52] = input[11]; + output[53] = input[43]; + output[54] = input[27]; + output[55] = input[59]; + output[56] = input[7]; + output[57] = input[39]; + output[58] = input[23]; + output[59] = input[55]; + output[60] = input[15]; + output[61] = input[47]; + output[62] = input[31]; + output[63] = input[63]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = output[20]; + step[21] = output[21]; + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = output[26]; + step[27] = output[27]; + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); + step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); + step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = step[10]; + output[11] = step[11]; + output[12] = step[12]; + output[13] = step[13]; + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); + output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); + output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); + output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); + step[35] = output[35]; + step[36] = output[36]; + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); + step[39] = output[39]; + step[40] = output[40]; + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); + step[43] = output[43]; + step[44] = output[44]; + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); + step[47] = output[47]; + step[48] = output[48]; + step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); + step[51] = output[51]; + step[52] = output[52]; + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); + step[55] = output[55]; + step[56] = output[56]; + step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); + step[59] = output[59]; + step[60] = output[60]; + step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); + step[63] = output[63]; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); + output[19] = step[19]; + output[20] = step[20]; + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); + output[23] = step[23]; + output[24] = step[24]; + output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); + output[27] = step[27]; + output[28] = step[28]; + output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); + step[38] = output[38]; + step[39] = output[39]; + step[40] = output[40]; + step[41] = output[41]; + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); + step[54] = output[54]; + step[55] = output[55]; + step[56] = output[56]; + step[57] = output[57]; + step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); + step[62] = output[62]; + step[63] = output[63]; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); + step[44] = output[44]; + step[45] = output[45]; + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = output[50]; + step[51] = output[51]; + step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); + step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); + step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); + step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); + step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); + step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); + step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = output[36]; + step[37] = output[37]; + step[38] = output[38]; + step[39] = output[39]; + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); + step[56] = output[56]; + step[57] = output[57]; + step[58] = output[58]; + step[59] = output[59]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); + output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); + output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); + output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); + output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); + output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); + output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); + output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); + output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); + output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct8Operator.cs new file mode 100644 index 000000000..ebb291fbb --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Dct8Operator.cs @@ -0,0 +1,238 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the eight-point discrete cosine inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the eight-point AV1 inverse discrete cosine transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply + /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. + /// + internal readonly struct Dct8Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative eight-point AV1 inverse discrete cosine transform. + /// + /// The eight frequency-domain coefficients. + /// The eight spatial-domain residual values. + /// The eight-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[4]; + output[2] = input[2]; + output[3] = input[6]; + output[4] = input[1]; + output[5] = input[5]; + output[6] = input[3]; + output[7] = input[7]; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs new file mode 100644 index 000000000..011d3a4c0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs @@ -0,0 +1,71 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sixteen-point identity inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the sixteen-point AV1 inverse identity transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, + /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. + /// + internal readonly struct Identity16Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative sixteen-point AV1 inverse identity transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization. + for (int i = 0; i < 16; i++) + { + output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity32Operator.cs new file mode 100644 index 000000000..68b547ab5 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity32Operator.cs @@ -0,0 +1,71 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the thirty-two-point identity inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the thirty-two-point AV1 inverse identity transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, + /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. + /// + internal readonly struct Identity32Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative thirty-two-point AV1 inverse identity transform. + /// + /// The thirty-two frequency-domain coefficients. + /// The thirty-two scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization. + for (int i = 0; i < 32; i++) + { + output[i] = input[i] * 4; + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs new file mode 100644 index 000000000..7113d8986 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs @@ -0,0 +1,71 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the four-point identity inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the four-point AV1 inverse identity transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, + /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. + /// + internal readonly struct Identity4Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative four-point AV1 inverse identity transform. + /// + /// The four frequency-domain coefficients. + /// The four scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization. + for (int i = 0; i < 4; i++) + { + output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity8Operator.cs new file mode 100644 index 000000000..645e58e2d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity8Operator.cs @@ -0,0 +1,71 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the eight-point identity inverse transform operator. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Defines the eight-point AV1 inverse identity transform operator. + /// + /// + /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, + /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. + /// + internal readonly struct Identity8Operator : IAv1InverseTransform1dOperator + { + /// + /// Applies the normative eight-point AV1 inverse identity transform. + /// + /// The eight frequency-domain coefficients. + /// The eight scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization. + for (int i = 0; i < 8; i++) + { + output[i] = input[i] * 2; + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs similarity index 82% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs index 23ddc7e2f..9d253ed3c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs @@ -4,7 +4,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -17,8 +16,101 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// cross-lane permutations. Reconstruction adds the final residuals to their matching prediction lanes before /// narrowing to the decoded sample depth. /// -internal static class Av1Inverse2dTransformer +internal static partial class Av1Inverse2dTransformer { + /// + /// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional inverse transform. + /// + /// + /// Each overload performs the same staged fixed-point inverse transform. In the SIMD overloads, each vector field identifies + /// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore + /// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage + /// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the + /// JIT to specialize every static interface call outside the stage network. + /// + internal interface IAv1InverseTransform1dOperator + { + /// + /// Transforms one axis when hardware vectorization is unavailable. + /// + /// The source values for the transform axis. + /// The destination values for the transform axis. + /// The fixed stage storage for the transform axis. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange); + + /// + /// Transforms four independent axes in parallel. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The fixed stage storage for four transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); + + /// + /// Transforms eight independent axes in parallel. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The fixed stage storage for eight transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); + } + + /// + /// Defines how inverse-transform residuals are added to a decoded sample representation. + /// + /// + /// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions, + /// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable + /// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize. + /// + /// The decoded sample storage type. + internal interface IAv1InverseTransformOutputOperator + where TSample : unmanaged + { + /// + /// Adds one residual to a predicted sample and clips the result to the coded bit depth. + /// + /// The predicted sample. + /// The inverse-transform residual. + /// The coded sample bit depth. + /// The reconstructed sample. + public static abstract TSample Add(TSample prediction, int residual, int bitDepth); + + /// + /// Adds four residuals to four predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The four inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth); + + /// + /// Adds eight residuals to eight predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The eight inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); + } + /// /// Applies an inverse transform and adds its residual to high-bit-depth predicted samples. /// @@ -39,7 +131,7 @@ internal static class Av1Inverse2dTransformer ref Av1Transform2dFlipConfiguration config, Span workspace, int bitDepth) - => Transform2dAdd>( + => Transform2dAdd>( input, outputForRead, strideForRead, @@ -67,7 +159,7 @@ internal static class Av1Inverse2dTransformer int strideForWrite, ref Av1Transform2dFlipConfiguration config, Span workspace) - => Transform2dAdd>( + => Transform2dAdd>( input, outputForRead, strideForRead, @@ -96,62 +188,62 @@ internal static class Av1Inverse2dTransformer switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct32: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct64: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity32: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; @@ -174,67 +266,67 @@ internal static class Av1Inverse2dTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1InverseTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct32: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct64: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity32: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; @@ -257,8 +349,8 @@ internal static class Av1Inverse2dTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1InverseTransform1dOperator + where TRowOperator : struct, IAv1InverseTransform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); @@ -309,8 +401,8 @@ internal static class Av1Inverse2dTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1InverseTransform1dOperator + where TRowOperator : struct, IAv1InverseTransform1dOperator { const int laneCount = 8; const int vectorLength = Av1Constants.MaxTransformSize * laneCount; @@ -465,8 +557,8 @@ internal static class Av1Inverse2dTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1InverseTransform1dOperator + where TRowOperator : struct, IAv1InverseTransform1dOperator { const int laneCount = 4; const int vectorLength = Av1Constants.MaxTransformSize * laneCount; @@ -597,8 +689,8 @@ internal static class Av1Inverse2dTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1InverseTransform1dOperator + where TRowOperator : struct, IAv1InverseTransform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.OutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.OutputOperator.cs new file mode 100644 index 000000000..738caeab6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.OutputOperator.cs @@ -0,0 +1,100 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Provides the sample-output operator shared by inverse transform traversals. +/// +internal static partial class Av1Inverse2dTransformer +{ + /// + /// Reconstructs AV1 samples from predicted values and inverse-transform residuals. + /// + /// + /// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to + /// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width + /// stores. The closed specialization removes storage-type branches from hot loops. + /// + /// The decoded sample storage type. + internal readonly struct OutputOperator : IAv1InverseTransformOutputOperator + where TSample : unmanaged + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TSample Add(TSample prediction, int residual, int bitDepth) + { + // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each + // closed transform so storage selection does not introduce a branch in the reconstruction loop. + if (typeof(TSample) == typeof(byte)) + { + byte value = (byte)Math.Clamp(Unsafe.As(ref prediction) + residual, byte.MinValue, byte.MaxValue); + return Unsafe.As(ref value); + } + + short result = (short)Math.Clamp(Unsafe.As(ref prediction) + residual, 0, (1 << bitDepth) - 1); + return Unsafe.As(ref result); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth) + { + if (typeof(TSample) == typeof(byte)) + { + // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach + // memory, which keeps reconstruction valid at a tightly packed row boundary. + ref byte source = ref Unsafe.As(ref prediction); + uint packed = Unsafe.ReadUnaligned(ref source); + Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32(); + Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue)); + Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt32().ToScalar()); + return; + } + + ref short highBitDepthSource = ref Unsafe.As(ref prediction); + ulong highBitDepthPacked = Unsafe.ReadUnaligned(ref Unsafe.As(ref highBitDepthSource)); + Vector128 highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16()); + Vector128 highBitDepthReconstructed = + Vector128.Clamp(highBitDepthPredicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1)); + + Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar()); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth) + { + if (typeof(TSample) == typeof(byte)) + { + // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store + // covers only those reconstructed samples and does not require destination padding. + ref byte source = ref Unsafe.As(ref prediction); + ulong packed = Unsafe.ReadUnaligned(ref source); + Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32(); + Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue)); + Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt64().ToScalar()); + return; + } + + ref short highBitDepthSource = ref Unsafe.As(ref prediction); + Vector256 highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource)); + Vector256 highBitDepthReconstructed = + Vector256.Clamp(highBitDepthPredicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1)); + + Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper()); + narrowed.StoreUnsafe(ref Unsafe.As(ref destination)); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs deleted file mode 100644 index 5b6fa5c93..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Common.Helpers; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Reconstructs AV1 samples from predicted values and inverse-transform residuals. -/// -/// -/// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to -/// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width -/// stores. The closed specialization removes storage-type branches from hot loops. -/// -/// The decoded sample storage type. -internal readonly struct Av1InverseTransformOutputOperator : IAv1InverseTransformOutputOperator - where TSample : unmanaged -{ - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static TSample Add(TSample prediction, int residual, int bitDepth) - { - // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each - // closed transform so storage selection does not introduce a branch in the reconstruction loop. - if (typeof(TSample) == typeof(byte)) - { - byte value = (byte)Math.Clamp(Unsafe.As(ref prediction) + residual, byte.MinValue, byte.MaxValue); - return Unsafe.As(ref value); - } - - short result = (short)Math.Clamp(Unsafe.As(ref prediction) + residual, 0, (1 << bitDepth) - 1); - return Unsafe.As(ref result); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth) - { - if (typeof(TSample) == typeof(byte)) - { - // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach - // memory, which keeps reconstruction valid at a tightly packed row boundary. - ref byte source = ref Unsafe.As(ref prediction); - uint packed = Unsafe.ReadUnaligned(ref source); - Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32(); - Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue)); - Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt32().ToScalar()); - return; - } - - ref short highBitDepthSource = ref Unsafe.As(ref prediction); - ulong highBitDepthPacked = Unsafe.ReadUnaligned(ref Unsafe.As(ref highBitDepthSource)); - Vector128 highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16()); - Vector128 highBitDepthReconstructed = - Vector128.Clamp(highBitDepthPredicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1)); - - Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar()); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth) - { - if (typeof(TSample) == typeof(byte)) - { - // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store - // covers only those reconstructed samples and does not require destination padding. - ref byte source = ref Unsafe.As(ref prediction); - ulong packed = Unsafe.ReadUnaligned(ref source); - Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32(); - Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue)); - Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt64().ToScalar()); - return; - } - - ref short highBitDepthSource = ref Unsafe.As(ref prediction); - Vector256 highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource)); - Vector256 highBitDepthReconstructed = - Vector256.Clamp(highBitDepthPredicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1)); - - Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper()); - narrowed.StoreUnsafe(ref Unsafe.As(ref destination)); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs index 034e1d5a4..386770c66 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs @@ -40,7 +40,7 @@ internal static class Av1InverseWalshHadamardTransformer int writeStride, int coefficientCount, Span workspace) - => TransformAdd>( + => TransformAdd>( coefficients, readBuffer, readStride, @@ -70,7 +70,7 @@ internal static class Av1InverseWalshHadamardTransformer int coefficientCount, Span workspace, int bitDepth) - => TransformAdd>( + => TransformAdd>( coefficients, readBuffer, readStride, @@ -93,7 +93,7 @@ internal static class Av1InverseWalshHadamardTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransformOutputOperator { if (Vector128.IsHardwareAccelerated) { @@ -116,7 +116,7 @@ internal static class Av1InverseWalshHadamardTransformer int coefficientCount, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransformOutputOperator { ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); Vector128 row0; @@ -172,7 +172,7 @@ internal static class Av1InverseWalshHadamardTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransformOutputOperator { ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer); ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer); @@ -300,7 +300,7 @@ internal static class Av1InverseWalshHadamardTransformer Vector128 row3, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TOutputOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransformOutputOperator { ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer); ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs deleted file mode 100644 index 3c354fd6c..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator. -/// -internal readonly struct Av1Adst16Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs deleted file mode 100644 index ec561c746..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the four-point AV1 forward asymmetric discrete sine transform operator. -/// -internal readonly struct Av1Adst4Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs deleted file mode 100644 index 1d7e0a424..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the eight-point AV1 forward asymmetric discrete sine transform operator. -/// -internal readonly struct Av1Adst8Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs deleted file mode 100644 index e1b447b15..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixteen-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct16Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs deleted file mode 100644 index 5749ab40e..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the thirty-two-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct32Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs deleted file mode 100644 index bef5993f3..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the four-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct4Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs deleted file mode 100644 index ca12aa88a..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixty-four-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct64Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs deleted file mode 100644 index 8fc637085..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the eight-point AV1 forward discrete cosine transform operator. -/// -internal readonly struct Av1Dct8Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs deleted file mode 100644 index 5216a8900..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the sixteen-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity16Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs deleted file mode 100644 index f15c0b868..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the thirty-two-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity32Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs deleted file mode 100644 index 89236df3d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the four-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity4Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs deleted file mode 100644 index 96a295221..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines the eight-point AV1 forward identity transform operator. -/// -internal readonly struct Av1Identity8Forward1dOperator : IAv1ForwardTransform1dOperator -{ - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs deleted file mode 100644 index 6aa68233d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type. -/// -/// -/// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type -/// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch -/// inside the transform stages. -/// -internal interface IAv1ForwardTransform1dOperator -{ - /// - /// Transforms the independent axes stored in each value lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static abstract void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct; -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs deleted file mode 100644 index 7e2244cc6..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Defines how inverse-transform residuals are added to a decoded sample representation. -/// -/// -/// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions, -/// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable -/// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize. -/// -/// The decoded sample storage type. -internal interface IAv1InverseTransformOutputOperator - where TSample : unmanaged -{ - /// - /// Adds one residual to a predicted sample and clips the result to the coded bit depth. - /// - /// The predicted sample. - /// The inverse-transform residual. - /// The coded sample bit depth. - /// The reconstructed sample. - public static abstract TSample Add(TSample prediction, int residual, int bitDepth); - - /// - /// Adds four residuals to four predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The four inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth); - - /// - /// Adds eight residuals to eight predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The eight inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs deleted file mode 100644 index 09ecb8a3b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional transform. -/// -/// -/// Each overload performs the same staged fixed-point transform. In the SIMD overloads, each vector field identifies -/// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore -/// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage -/// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the -/// JIT to specialize every static interface call outside the stage network. -/// -internal interface IAv1Transform1dOperator -{ - /// - /// Transforms one axis when hardware vectorization is unavailable. - /// - /// The source values for the transform axis. - /// The destination values for the transform axis. - /// The fixed stage storage for the transform axis. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange); - - /// - /// Transforms four independent axes in parallel. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed stage storage for four transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); - - /// - /// Transforms eight independent axes in parallel. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed stage storage for eight transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs deleted file mode 100644 index 04f0aff06..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Adst16Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen spatial-domain residual values. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[15]; - output[1] = input[0]; - output[2] = input[13]; - output[3] = input[2]; - output[4] = input[11]; - output[5] = input[4]; - output[6] = input[9]; - output[7] = input[6]; - output[8] = input[7]; - output[9] = input[8]; - output[10] = input[5]; - output[11] = input[10]; - output[12] = input[3]; - output[13] = input[12]; - output[14] = input[1]; - output[15] = input[14]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[8]; - output[2] = step[12]; - output[3] = -step[4]; - output[4] = step[6]; - output[5] = -step[14]; - output[6] = step[10]; - output[7] = -step[2]; - output[8] = step[3]; - output[9] = -step[11]; - output[10] = step[15]; - output[11] = -step[7]; - output[12] = step[5]; - output[13] = -step[13]; - output[14] = step[9]; - output[15] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V15; - output.V1 = input.V0; - output.V2 = input.V13; - output.V3 = input.V2; - output.V4 = input.V11; - output.V5 = input.V4; - output.V6 = input.V9; - output.V7 = input.V6; - output.V8 = input.V7; - output.V9 = input.V8; - output.V10 = input.V5; - output.V11 = input.V10; - output.V12 = input.V3; - output.V13 = input.V12; - output.V14 = input.V1; - output.V15 = input.V14; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V8; - output.V2 = step.V12; - output.V3 = -step.V4; - output.V4 = step.V6; - output.V5 = -step.V14; - output.V6 = step.V10; - output.V7 = -step.V2; - output.V8 = step.V3; - output.V9 = -step.V11; - output.V10 = step.V15; - output.V11 = -step.V7; - output.V12 = step.V5; - output.V13 = -step.V13; - output.V14 = step.V9; - output.V15 = -step.V1; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V15; - output.V1 = input.V0; - output.V2 = input.V13; - output.V3 = input.V2; - output.V4 = input.V11; - output.V5 = input.V4; - output.V6 = input.V9; - output.V7 = input.V6; - output.V8 = input.V7; - output.V9 = input.V8; - output.V10 = input.V5; - output.V11 = input.V10; - output.V12 = input.V3; - output.V13 = input.V12; - output.V14 = input.V1; - output.V15 = input.V14; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V8; - output.V2 = step.V12; - output.V3 = -step.V4; - output.V4 = step.V6; - output.V5 = -step.V14; - output.V6 = step.V10; - output.V7 = -step.V2; - output.V8 = step.V3; - output.V9 = -step.V11; - output.V10 = step.V15; - output.V11 = -step.V7; - output.V12 = step.V5; - output.V13 = -step.V13; - output.V14 = step.V9; - output.V15 = -step.V1; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs deleted file mode 100644 index fef59237f..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Adst4Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform. - /// - /// The four frequency-domain coefficients. - /// The four spatial-domain residual values. - /// The stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the sine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - - // libaom widens the complete four-point factorization because the products retain their fixed-point scale - // until the final shift. The stage buffer is therefore unnecessary for this transform size. - long x0 = input[0]; - long x1 = input[1]; - long x2 = input[2]; - long x3 = input[3]; - - _ = step; - _ = stageRange; - - // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel. - if ((x0 | x1 | x2 | x3) == 0) - { - output[..4].Clear(); - return; - } - - // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3. - long s0 = sinpi[1] * x0; - long s1 = sinpi[2] * x0; - long s2 = sinpi[3] * x1; - long s3 = sinpi[4] * x2; - long s4 = sinpi[1] * x2; - long s5 = sinpi[2] * x3; - long s6 = sinpi[4] * x3; - long s7 = (x0 - x2) + x3; - - // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding. - s0 += s3; - s1 -= s4; - s3 = s2; - s2 = sinpi[3] * s7; - s0 += s5; - s1 -= s6; - x0 = s0 + s3; - x1 = s1 + s3; - x2 = s2; - x3 = (s0 + s1) - s3; - - output[0] = Av1Math.RoundShift(x0, cosBit); - output[1] = Av1Math.RoundShift(x1, cosBit); - output[2] = Av1Math.RoundShift(x2, cosBit); - output[3] = Av1Math.RoundShift(x3, cosBit); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - /// Applies the inverse four-point matrix to four independent axes. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector128 x0 = input.V0; - Vector128 x1 = input.V1; - Vector128 x2 = input.V2; - Vector128 x3 = input.V3; - - // The products retain the sine-table scale across the complete matrix. The bounded transform inputs make - // the optimized kernels' wrapping 32-bit multiply/add sequence valid until the terminal rounding shift. - output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the inverse four-point matrix to eight independent axes. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector256 x0 = input.V0; - Vector256 x1 = input.V1; - Vector256 x2 = input.V2; - Vector256 x3 = input.V3; - - output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs deleted file mode 100644 index d6978dadf..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Adst8Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform. - /// - /// The eight frequency-domain coefficients. - /// The eight spatial-domain residual values. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[7]; - output[1] = input[0]; - output[2] = input[5]; - output[3] = input[2]; - output[4] = input[3]; - output[5] = input[4]; - output[6] = input[1]; - output[7] = input[6]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[4]; - output[2] = step[6]; - output[3] = -step[2]; - output[4] = step[3]; - output[5] = -step[7]; - output[6] = step[5]; - output[7] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V7; - output.V1 = input.V0; - output.V2 = input.V5; - output.V3 = input.V2; - output.V4 = input.V3; - output.V5 = input.V4; - output.V6 = input.V1; - output.V7 = input.V6; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V4; - output.V2 = step.V6; - output.V3 = -step.V2; - output.V4 = step.V3; - output.V5 = -step.V7; - output.V6 = step.V5; - output.V7 = -step.V1; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V7; - output.V1 = input.V0; - output.V2 = input.V5; - output.V3 = input.V2; - output.V4 = input.V3; - output.V5 = input.V4; - output.V6 = input.V1; - output.V7 = input.V6; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V4; - output.V2 = step.V6; - output.V3 = -step.V2; - output.V4 = step.V3; - output.V5 = -step.V7; - output.V6 = step.V5; - output.V7 = -step.V1; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs deleted file mode 100644 index 9659fd51e..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs +++ /dev/null @@ -1,475 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 16-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Dct16Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 16-point AV1 inverse discrete cosine transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen spatial-domain residual values. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[8]; - output[2] = input[4]; - output[3] = input[12]; - output[4] = input[2]; - output[5] = input[10]; - output[6] = input[6]; - output[7] = input[14]; - output[8] = input[1]; - output[9] = input[9]; - output[10] = input[5]; - output[11] = input[13]; - output[12] = input[3]; - output[13] = input[11]; - output[14] = input[7]; - output[15] = input[15]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V8; - output.V2 = input.V4; - output.V3 = input.V12; - output.V4 = input.V2; - output.V5 = input.V10; - output.V6 = input.V6; - output.V7 = input.V14; - output.V8 = input.V1; - output.V9 = input.V9; - output.V10 = input.V5; - output.V11 = input.V13; - output.V12 = input.V3; - output.V13 = input.V11; - output.V14 = input.V7; - output.V15 = input.V15; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V8; - output.V2 = input.V4; - output.V3 = input.V12; - output.V4 = input.V2; - output.V5 = input.V10; - output.V6 = input.V6; - output.V7 = input.V14; - output.V8 = input.V1; - output.V9 = input.V9; - output.V10 = input.V5; - output.V11 = input.V13; - output.V12 = input.V3; - output.V13 = input.V11; - output.V14 = input.V7; - output.V15 = input.V15; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs deleted file mode 100644 index 62eaadbb9..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs +++ /dev/null @@ -1,1027 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 32-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Dct32Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 32-point AV1 inverse discrete cosine transform. - /// - /// The 32 frequency-domain coefficients. - /// The 32 spatial-domain residual values. - /// The 32-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[16]; - output[2] = input[8]; - output[3] = input[24]; - output[4] = input[4]; - output[5] = input[20]; - output[6] = input[12]; - output[7] = input[28]; - output[8] = input[2]; - output[9] = input[18]; - output[10] = input[10]; - output[11] = input[26]; - output[12] = input[6]; - output[13] = input[22]; - output[14] = input[14]; - output[15] = input[30]; - output[16] = input[1]; - output[17] = input[17]; - output[18] = input[9]; - output[19] = input[25]; - output[20] = input[5]; - output[21] = input[21]; - output[22] = input[13]; - output[23] = input[29]; - output[24] = input[3]; - output[25] = input[19]; - output[26] = input[11]; - output[27] = input[27]; - output[28] = input[7]; - output[29] = input[23]; - output[30] = input[15]; - output[31] = input[31]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); - output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); - output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); - output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); - output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); - output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); - output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); - output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); - output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); - step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); - step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); - step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); - step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); - step[31] = output[31]; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); - output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); - output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); - output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); - output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); - output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); - output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); - step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); - step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); - step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); - output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); - output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); - output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); - output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); - step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); - step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); - step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); - output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); - output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); - output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V16; - output.V2 = input.V8; - output.V3 = input.V24; - output.V4 = input.V4; - output.V5 = input.V20; - output.V6 = input.V12; - output.V7 = input.V28; - output.V8 = input.V2; - output.V9 = input.V18; - output.V10 = input.V10; - output.V11 = input.V26; - output.V12 = input.V6; - output.V13 = input.V22; - output.V14 = input.V14; - output.V15 = input.V30; - output.V16 = input.V1; - output.V17 = input.V17; - output.V18 = input.V9; - output.V19 = input.V25; - output.V20 = input.V5; - output.V21 = input.V21; - output.V22 = input.V13; - output.V23 = input.V29; - output.V24 = input.V3; - output.V25 = input.V19; - output.V26 = input.V11; - output.V27 = input.V27; - output.V28 = input.V7; - output.V29 = input.V23; - output.V30 = input.V15; - output.V31 = input.V31; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); - step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); - step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); - output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); - output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); - output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); - output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); - output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); - output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); - output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); - output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); - output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); - output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); - step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); - step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); - step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); - step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); - step.V16 = output.V16; - step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); - step.V31 = output.V31; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - output.V8 = step.V8; - output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); - output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); - output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); - output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); - output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); - output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); - output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); - step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); - step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); - step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); - output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); - output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); - output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); - output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); - step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); - step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); - step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); - output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); - output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); - output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V16; - output.V2 = input.V8; - output.V3 = input.V24; - output.V4 = input.V4; - output.V5 = input.V20; - output.V6 = input.V12; - output.V7 = input.V28; - output.V8 = input.V2; - output.V9 = input.V18; - output.V10 = input.V10; - output.V11 = input.V26; - output.V12 = input.V6; - output.V13 = input.V22; - output.V14 = input.V14; - output.V15 = input.V30; - output.V16 = input.V1; - output.V17 = input.V17; - output.V18 = input.V9; - output.V19 = input.V25; - output.V20 = input.V5; - output.V21 = input.V21; - output.V22 = input.V13; - output.V23 = input.V29; - output.V24 = input.V3; - output.V25 = input.V19; - output.V26 = input.V11; - output.V27 = input.V27; - output.V28 = input.V7; - output.V29 = input.V23; - output.V30 = input.V15; - output.V31 = input.V31; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); - step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); - step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); - output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); - output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); - output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); - output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); - output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); - output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); - output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); - output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); - output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); - output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); - step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); - step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); - step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); - step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); - step.V16 = output.V16; - step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); - step.V31 = output.V31; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - output.V8 = step.V8; - output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); - output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); - output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); - output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); - output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); - output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); - output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); - step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); - step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); - step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); - output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); - output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); - output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); - output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); - step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); - step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); - step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); - output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); - output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); - output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs deleted file mode 100644 index 9b6705fff..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the four-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Dct4Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative four-point AV1 inverse discrete cosine transform. - /// - /// The four frequency-domain coefficients. - /// The four spatial-domain residual values. - /// The four-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output[0] = input[0]; - output[1] = input[2]; - output[2] = input[1]; - output[3] = input[3]; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output.V0 = input.V0; - output.V1 = input.V2; - output.V2 = input.V1; - output.V3 = input.V3; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output.V0 = input.V0; - output.V1 = input.V2; - output.V2 = input.V1; - output.V3 = input.V3; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs deleted file mode 100644 index 2ab33cb19..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs +++ /dev/null @@ -1,2272 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the 64-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Dct64Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative 64-point AV1 inverse discrete cosine transform. - /// - /// The 64 frequency-domain coefficients. - /// The 64 spatial-domain residual values. - /// The 64-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[32]; - output[2] = input[16]; - output[3] = input[48]; - output[4] = input[8]; - output[5] = input[40]; - output[6] = input[24]; - output[7] = input[56]; - output[8] = input[4]; - output[9] = input[36]; - output[10] = input[20]; - output[11] = input[52]; - output[12] = input[12]; - output[13] = input[44]; - output[14] = input[28]; - output[15] = input[60]; - output[16] = input[2]; - output[17] = input[34]; - output[18] = input[18]; - output[19] = input[50]; - output[20] = input[10]; - output[21] = input[42]; - output[22] = input[26]; - output[23] = input[58]; - output[24] = input[6]; - output[25] = input[38]; - output[26] = input[22]; - output[27] = input[54]; - output[28] = input[14]; - output[29] = input[46]; - output[30] = input[30]; - output[31] = input[62]; - output[32] = input[1]; - output[33] = input[33]; - output[34] = input[17]; - output[35] = input[49]; - output[36] = input[9]; - output[37] = input[41]; - output[38] = input[25]; - output[39] = input[57]; - output[40] = input[5]; - output[41] = input[37]; - output[42] = input[21]; - output[43] = input[53]; - output[44] = input[13]; - output[45] = input[45]; - output[46] = input[29]; - output[47] = input[61]; - output[48] = input[3]; - output[49] = input[35]; - output[50] = input[19]; - output[51] = input[51]; - output[52] = input[11]; - output[53] = input[43]; - output[54] = input[27]; - output[55] = input[59]; - output[56] = input[7]; - output[57] = input[39]; - output[58] = input[23]; - output[59] = input[55]; - output[60] = input[15]; - output[61] = input[47]; - output[62] = input[31]; - output[63] = input[63]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); - output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); - step[63] = output[63]; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); - step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); - step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); - step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); - step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); - step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); - step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); - output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); - output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); - output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); - output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); - output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); - output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); - output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); - output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); - output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V32; - output.V2 = input.V16; - output.V3 = input.V48; - output.V4 = input.V8; - output.V5 = input.V40; - output.V6 = input.V24; - output.V7 = input.V56; - output.V8 = input.V4; - output.V9 = input.V36; - output.V10 = input.V20; - output.V11 = input.V52; - output.V12 = input.V12; - output.V13 = input.V44; - output.V14 = input.V28; - output.V15 = input.V60; - output.V16 = input.V2; - output.V17 = input.V34; - output.V18 = input.V18; - output.V19 = input.V50; - output.V20 = input.V10; - output.V21 = input.V42; - output.V22 = input.V26; - output.V23 = input.V58; - output.V24 = input.V6; - output.V25 = input.V38; - output.V26 = input.V22; - output.V27 = input.V54; - output.V28 = input.V14; - output.V29 = input.V46; - output.V30 = input.V30; - output.V31 = input.V62; - output.V32 = input.V1; - output.V33 = input.V33; - output.V34 = input.V17; - output.V35 = input.V49; - output.V36 = input.V9; - output.V37 = input.V41; - output.V38 = input.V25; - output.V39 = input.V57; - output.V40 = input.V5; - output.V41 = input.V37; - output.V42 = input.V21; - output.V43 = input.V53; - output.V44 = input.V13; - output.V45 = input.V45; - output.V46 = input.V29; - output.V47 = input.V61; - output.V48 = input.V3; - output.V49 = input.V35; - output.V50 = input.V19; - output.V51 = input.V51; - output.V52 = input.V11; - output.V53 = input.V43; - output.V54 = input.V27; - output.V55 = input.V59; - output.V56 = input.V7; - output.V57 = input.V39; - output.V58 = input.V23; - output.V59 = input.V55; - output.V60 = input.V15; - output.V61 = input.V47; - output.V62 = input.V31; - output.V63 = input.V63; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = output.V21; - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = output.V26; - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); - step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); - step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = step.V10; - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = step.V13; - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); - output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); - output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); - step.V43 = output.V43; - step.V44 = output.V44; - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); - step.V51 = output.V51; - step.V52 = output.V52; - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); - step.V63 = output.V63; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); - output.V19 = step.V19; - output.V20 = step.V20; - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); - output.V27 = step.V27; - output.V28 = step.V28; - output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = output.V41; - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); - step.V54 = output.V54; - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); - output.V22 = step.V22; - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = step.V25; - output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); - step.V44 = output.V44; - step.V45 = output.V45; - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = output.V50; - step.V51 = output.V51; - step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = step.V18; - output.V19 = step.V19; - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V28 = step.V28; - output.V29 = step.V29; - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); - step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); - step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); - step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); - step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); - step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); - step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = output.V37; - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = output.V58; - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); - output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); - output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); - output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); - output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); - output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); - output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); - output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); - output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); - output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V32; - output.V2 = input.V16; - output.V3 = input.V48; - output.V4 = input.V8; - output.V5 = input.V40; - output.V6 = input.V24; - output.V7 = input.V56; - output.V8 = input.V4; - output.V9 = input.V36; - output.V10 = input.V20; - output.V11 = input.V52; - output.V12 = input.V12; - output.V13 = input.V44; - output.V14 = input.V28; - output.V15 = input.V60; - output.V16 = input.V2; - output.V17 = input.V34; - output.V18 = input.V18; - output.V19 = input.V50; - output.V20 = input.V10; - output.V21 = input.V42; - output.V22 = input.V26; - output.V23 = input.V58; - output.V24 = input.V6; - output.V25 = input.V38; - output.V26 = input.V22; - output.V27 = input.V54; - output.V28 = input.V14; - output.V29 = input.V46; - output.V30 = input.V30; - output.V31 = input.V62; - output.V32 = input.V1; - output.V33 = input.V33; - output.V34 = input.V17; - output.V35 = input.V49; - output.V36 = input.V9; - output.V37 = input.V41; - output.V38 = input.V25; - output.V39 = input.V57; - output.V40 = input.V5; - output.V41 = input.V37; - output.V42 = input.V21; - output.V43 = input.V53; - output.V44 = input.V13; - output.V45 = input.V45; - output.V46 = input.V29; - output.V47 = input.V61; - output.V48 = input.V3; - output.V49 = input.V35; - output.V50 = input.V19; - output.V51 = input.V51; - output.V52 = input.V11; - output.V53 = input.V43; - output.V54 = input.V27; - output.V55 = input.V59; - output.V56 = input.V7; - output.V57 = input.V39; - output.V58 = input.V23; - output.V59 = input.V55; - output.V60 = input.V15; - output.V61 = input.V47; - output.V62 = input.V31; - output.V63 = input.V63; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = output.V21; - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = output.V26; - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); - step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); - step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = step.V10; - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = step.V13; - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); - output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); - output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); - step.V43 = output.V43; - step.V44 = output.V44; - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); - step.V51 = output.V51; - step.V52 = output.V52; - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); - step.V63 = output.V63; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); - output.V19 = step.V19; - output.V20 = step.V20; - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); - output.V27 = step.V27; - output.V28 = step.V28; - output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = output.V41; - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); - step.V54 = output.V54; - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); - output.V22 = step.V22; - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = step.V25; - output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); - step.V44 = output.V44; - step.V45 = output.V45; - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = output.V50; - step.V51 = output.V51; - step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = step.V18; - output.V19 = step.V19; - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V28 = step.V28; - output.V29 = step.V29; - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); - step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); - step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); - step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); - step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); - step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); - step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = output.V37; - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = output.V58; - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); - output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); - output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); - output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); - output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); - output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); - output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); - output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); - output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); - output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs deleted file mode 100644 index 49df84124..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the eight-point AV1 inverse discrete cosine transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply -/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. -/// -internal readonly partial struct Av1Dct8Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative eight-point AV1 inverse discrete cosine transform. - /// - /// The eight frequency-domain coefficients. - /// The eight spatial-domain residual values. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[4]; - output[2] = input[2]; - output[3] = input[6]; - output[4] = input[1]; - output[5] = input[5]; - output[6] = input[3]; - output[7] = input[7]; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V4; - output.V2 = input.V2; - output.V3 = input.V6; - output.V4 = input.V1; - output.V5 = input.V5; - output.V6 = input.V3; - output.V7 = input.V7; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V4; - output.V2 = input.V2; - output.V3 = input.V6; - output.V4 = input.V1; - output.V5 = input.V5; - output.V6 = input.V3; - output.V7 = input.V7; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs deleted file mode 100644 index 8bdcb5bda..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the sixteen-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly partial struct Av1Identity16Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative sixteen-point AV1 inverse identity transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization. - for (int i = 0; i < 16; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs deleted file mode 100644 index 7af8b05e7..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the thirty-two-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly partial struct Av1Identity32Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative thirty-two-point AV1 inverse identity transform. - /// - /// The thirty-two frequency-domain coefficients. - /// The thirty-two scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization. - for (int i = 0; i < 32; i++) - { - output[i] = input[i] * 4; - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs deleted file mode 100644 index d9abda95b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the four-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly partial struct Av1Identity4Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative four-point AV1 inverse identity transform. - /// - /// The four frequency-domain coefficients. - /// The four scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization. - for (int i = 0; i < 4; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs deleted file mode 100644 index 0a9a33927..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Defines the eight-point AV1 inverse identity transform operator. -/// -/// -/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, -/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. -/// -internal readonly partial struct Av1Identity8Inverse1dOperator : IAv1Transform1dOperator -{ - /// - /// Applies the normative eight-point AV1 inverse identity transform. - /// - /// The eight frequency-domain coefficients. - /// The eight scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization. - for (int i = 0; i < 8; i++) - { - output[i] = input[i] * 2; - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index fc0ba961e..aca677eac 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -5,7 +5,6 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -40,18 +39,18 @@ public class Av1ForwardTransformTests [Fact] public void OneDimensionalOperatorsMatchAnalyticalReference() { - AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); } /// @@ -88,18 +87,18 @@ public class Av1ForwardTransformTests /// private static void AssertOneDimensionalOperators() { - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(32); - AssertOperator(64); - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(32); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); + AssertOperator(64); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); } /// @@ -108,7 +107,7 @@ public class Av1ForwardTransformTests /// The transform operator. /// The transform length. private static void AssertOperator(int length) - where TOperator : struct, IAv1ForwardTransform1dOperator + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { const int cosBit = 12; @@ -144,7 +143,7 @@ public class Av1ForwardTransformTests /// The analytical transform definition. /// The transform length. private static void AssertOperatorAccuracy(Av1TransformType1d transformType, int length) - where TOperator : struct, IAv1ForwardTransform1dOperator + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { const int cosBit = 13; const int testBlockCount = 500; @@ -191,7 +190,7 @@ public class Av1ForwardTransformTests /// The transform length. /// The fixed-point precision of the cosine constants. private static void AssertInt32Operator(int length, int cosBit) - where TOperator : struct, IAv1ForwardTransform1dOperator + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator where TVector : struct { int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(int); @@ -245,7 +244,7 @@ public class Av1ForwardTransformTests /// The transform length. /// The fixed-point precision of the cosine constants. private static void AssertInt16Operator(int length, int cosBit) - where TOperator : struct, IAv1ForwardTransform1dOperator + where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator where TVector : struct { int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(short); @@ -368,40 +367,40 @@ public class Av1ForwardTransformTests switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; } } @@ -415,45 +414,45 @@ public class Av1ForwardTransformTests /// The destination reference coefficients. /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceRow(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TColumnOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; } } @@ -472,8 +471,8 @@ public class Av1ForwardTransformTests int stride, Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1ForwardTransform1dOperator - where TRowOperator : struct, IAv1ForwardTransform1dOperator + where TColumnOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TRowOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1FrameBufferTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1FrameBufferTests.cs new file mode 100644 index 000000000..37f356af0 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1FrameBufferTests.cs @@ -0,0 +1,132 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.Tests.Memory; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies AV1 frame and block-decoder allocation ownership during constructor rollback. +/// +[Trait("Format", "Avif")] +public class Av1FrameBufferTests +{ + /// + /// Verifies that a failure while renting the final chroma plane releases every previously rented plane. + /// + [Fact] + public void ConstructorFailureOnThirdPlaneReleasesEarlierPlanes() + { + FailingTestMemoryAllocator allocator = new(failureAllocationNumber: 3); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + ObuSequenceHeader sequenceHeader = new() + { + MaxFrameWidth = 1, + MaxFrameHeight = 1, + ColorConfig = new ObuColorConfig + { + IsMonochrome = false, + SubSamplingX = true, + SubSamplingY = true, + BitDepth = Av1BitDepth.EightBit + } + }; + + // All three padded planes fit in one backing owner each, making attempt three the Cr plane rent after Y and + // Cb have succeeded. The allocator log therefore contains exactly the two owners requiring rollback. + + Assert.Throws( + () => new Av1FrameBuffer(configuration, sequenceHeader, Av1ColorFormat.Yuv420, false)); + + Assert.Equal(3, allocator.AllocationAttemptCount); + Assert.Equal(2, allocator.AllocationLog.Count); + Assert.Equal(2, allocator.ReturnLog.Count); + Assert.Equal(allocator.AllocationLog[0].HashCodeOfBuffer, allocator.ReturnLog[0].HashCodeOfBuffer); + Assert.Equal(allocator.AllocationLog[1].HashCodeOfBuffer, allocator.ReturnLog[1].HashCodeOfBuffer); + } + + /// + /// Verifies that a block-decoder workspace failure releases every workspace rented earlier in construction. + /// + [Fact] + public void BlockDecoderConstructorFailureReleasesEarlierWorkspaces() + { + FailingTestMemoryAllocator allocator = new(failureAllocationNumber: 3); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + ObuSequenceHeader sequenceHeader = new() + { + MaxFrameWidth = 64, + MaxFrameHeight = 64, + Use128x128Superblock = false, + ColorConfig = new ObuColorConfig + { + IsMonochrome = true, + BitDepth = Av1BitDepth.EightBit + } + }; + + using Av1FrameBuffer frameBuffer = new(configuration, sequenceHeader, Av1ColorFormat.Yuv400, false); + ObuFrameHeader frameHeader = new(); + Av1LoopFilterContext loopFilterContext = new(sequenceHeader); + Av1InverseQuantizer inverseQuantizer = new(sequenceHeader, frameHeader); + + // The frame's luma plane is allocation attempt one. Resetting only the logs preserves that counter, so the + // inverse-quantization workspace succeeds on attempt two and the transform workspace fails on attempt three. + allocator.EnableNonThreadSafeLogging(); + + Assert.Throws( + () => new Av1BlockDecoder(sequenceHeader, frameHeader, frameBuffer, loopFilterContext, inverseQuantizer)); + + Assert.Equal(3, allocator.AllocationAttemptCount); + Assert.Single(allocator.AllocationLog); + Assert.Single(allocator.ReturnLog); + Assert.Equal(allocator.AllocationLog[0].HashCodeOfBuffer, allocator.ReturnLog[0].HashCodeOfBuffer); + } + + /// + /// Provides tracked plane owners until the configured allocation attempt fails. + /// + private sealed class FailingTestMemoryAllocator : TestMemoryAllocator + { + private readonly int failureAllocationNumber; + private int allocationAttemptCount; + + /// + /// Initializes a new instance of the class. + /// + /// The one-based allocation attempt that must fail. + public FailingTestMemoryAllocator(int failureAllocationNumber) + { + this.failureAllocationNumber = failureAllocationNumber; + this.EnableNonThreadSafeLogging(); + } + + /// + /// Gets the number of backing-owner allocation attempts made through this allocator. + /// + public int AllocationAttemptCount => this.allocationAttemptCount; + + /// + protected override AllocationTrackedMemoryManager AllocateCore(int length, AllocationOptions options = AllocationOptions.None) + { + this.allocationAttemptCount++; + + if (this.allocationAttemptCount == this.failureAllocationNumber) + { + // Fail before delegating so this attempt never creates or tracks an owner. Diagnostic counts then + // describe only the successfully published owners that the constructor is responsible for releasing. + throw new InvalidMemoryOperationException("The configured AV1 allocation failed."); + } + + return base.AllocateCore(length, options); + } + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraBlockCopyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraBlockCopyTests.cs index 5dc2c1943..124eb9a27 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraBlockCopyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraBlockCopyTests.cs @@ -35,7 +35,7 @@ public class Av1IntraBlockCopyTests int[] weights = new int[8]; Av1MotionVector actual = Av1IntraBlockCopy.FindReference( - partitionInfo, + ref partitionInfo, tileInfo, sequenceHeader.SuperblockModeInfoSize, candidates, @@ -71,7 +71,7 @@ public class Av1IntraBlockCopyTests int[] weights = new int[8]; Av1MotionVector actual = Av1IntraBlockCopy.FindReference( - partitionInfo, + ref partitionInfo, tileInfo, sequenceHeader.SuperblockModeInfoSize, candidates, @@ -99,13 +99,13 @@ public class Av1IntraBlockCopyTests Av1TileInfo tileInfo = CreateTileInfo(); // A source five 64-sample columns earlier satisfies both the four-column delay and same-row wavefront limit. - Assert.True(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -2560), partitionInfo, tileInfo, sequenceHeader)); + Assert.True(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -2560), ref partitionInfo, tileInfo, sequenceHeader)); // Moving the source one 64-sample column to the right reaches the forbidden delay boundary exactly. - Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -2048), partitionInfo, tileInfo, sequenceHeader)); - Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -2559), partitionInfo, tileInfo, sequenceHeader)); - Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -4608), partitionInfo, tileInfo, sequenceHeader)); - Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(512, -2560), partitionInfo, tileInfo, sequenceHeader)); + Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -2048), ref partitionInfo, tileInfo, sequenceHeader)); + Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -2559), ref partitionInfo, tileInfo, sequenceHeader)); + Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(0, -4608), ref partitionInfo, tileInfo, sequenceHeader)); + Assert.False(Av1IntraBlockCopy.IsValid(new Av1MotionVector(512, -2560), ref partitionInfo, tileInfo, sequenceHeader)); } /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 9f15d1c04..621c0f3d5 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -4,8 +4,6 @@ using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -48,11 +46,11 @@ public class Av1InverseTransformTests /// private static void AssertDctOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); - AssertOperatorParity(64); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); + AssertOperatorParity(32); + AssertOperatorParity(64); } /// @@ -60,9 +58,9 @@ public class Av1InverseTransformTests /// private static void AssertAdstOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); } /// @@ -70,10 +68,10 @@ public class Av1InverseTransformTests /// private static void AssertIdentityOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); + AssertOperatorParity(32); } [Theory] @@ -136,18 +134,18 @@ public class Av1InverseTransformTests [Fact] public void ForwardAndInverseOperatorPairsReconstructTheirInput() { - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size8x8, 2, 2); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size16x16, 3, 3); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size32x32, 4, 4); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size64x64, 5, 5); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size8x8, 2, 2); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size16x16, 3, 3); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size8x8, 2, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size16x16, 3, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size8x8, 2, 2); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size16x16, 3, 3); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size32x32, 4, 4); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size64x64, 5, 5); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size8x8, 2, 2); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size16x16, 3, 3); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size8x8, 2, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size16x16, 3, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); } /// @@ -464,7 +462,7 @@ public class Av1InverseTransformTests /// The inverse transform operator. /// The transform length. private static void AssertOperatorParity(int length) - where TOperator : struct, IAv1Transform1dOperator + where TOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator { const int cosBit = 12; Av1TransformStageRange stageRange = default; @@ -538,8 +536,8 @@ public class Av1InverseTransformTests /// The power-of-two scale applied by the operator pair. /// The maximum permitted reconstruction error. private static void AssertRoundTrip(Av1TransformType transformType, Av1TransformSize transformSize, int scaleLog2, int allowedError) - where TForwardOperator : struct, IAv1ForwardTransform1dOperator - where TInverseOperator : struct, IAv1Transform1dOperator + where TForwardOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TInverseOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator { const int bitDepth = 10; const int testBlockCount = 30; @@ -598,40 +596,40 @@ public class Av1InverseTransformTests switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; default: Assert.Fail($"Unexpected column function {config.TransformFunctionTypeColumn} for {transformType} {transformSize}."); @@ -652,45 +650,45 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct32: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct64: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity32: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; default: Assert.Fail($"Unexpected row function {config.TransformFunctionTypeRow} for {transformType} {transformSize}."); @@ -712,8 +710,8 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator + where TRowOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -763,8 +761,8 @@ public class Av1InverseTransformTests int[] coefficients, Av1TransformSize transformSize, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator + where TRowOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator { const int bitDepth = 8; int width = transformSize.GetWidth(); @@ -789,10 +787,10 @@ public class Av1InverseTransformTests Array.Fill(scalar, byte.MaxValue); Array.Fill(vector128, byte.MaxValue); - Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); - Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); @@ -803,7 +801,7 @@ public class Av1InverseTransformTests int[] vector256Workspace = new int[workspaceLength]; Array.Fill(vector256, byte.MaxValue); - Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); Assert.Equal(scalar, vector256); @@ -824,8 +822,8 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator + where TRowOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -850,10 +848,10 @@ public class Av1InverseTransformTests Array.Fill(scalar, short.MinValue); Array.Fill(vector128, short.MinValue); - Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); - Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); @@ -864,7 +862,7 @@ public class Av1InverseTransformTests int[] vector256Workspace = new int[workspaceLength]; Array.Fill(vector256, short.MinValue); - Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); Assert.Equal(scalar, vector256); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReconstructionConformanceTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReconstructionConformanceTests.cs index 8241a5fa0..39ca33fd0 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReconstructionConformanceTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReconstructionConformanceTests.cs @@ -214,6 +214,58 @@ public class Av1ReconstructionConformanceTests public void DecodeWithPaletteMatchesPinnedLibaomReference() => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidatePaletteNativeFixture, PaletteConfigurations); + /// + /// Verifies that a real palette frame whose tile entropy payload ends early is rejected instead of being decoded + /// from the range decoder's implicit zero padding. + /// + [Fact] + public void DecodeFrameBufferRejectsTruncatedPaletteTileEntropy() + { + const int TruncatedTileByteCount = 8; + byte[] validPayload = TestFile.Create(TestImages.Heif.Av1Palette8BitPayload).Bytes; + int obuOffset = 0; + int finalObuOffset = 0; + int finalSizeFieldOffset = 0; + int finalSizeFieldLength = 0; + ulong finalPayloadLength = 0; + while (obuOffset < validPayload.Length) + { + byte obuHeader = validPayload[obuOffset]; + Assert.True((obuHeader & 0x02) != 0); + + int headerLength = 1 + ((obuHeader >> 2) & 1); + int sizeFieldOffset = obuOffset + headerLength; + Av1BitStreamReader sizeReader = new(validPayload.AsSpan(sizeFieldOffset)); + ulong payloadLength = sizeReader.ReadLittleEndianBytes128(out int sizeFieldLength); + int nextObuOffset = checked(sizeFieldOffset + sizeFieldLength + (int)payloadLength); + if (nextObuOffset == validPayload.Length) + { + finalObuOffset = obuOffset; + finalSizeFieldOffset = sizeFieldOffset; + finalSizeFieldLength = sizeFieldLength; + finalPayloadLength = payloadLength; + } + + obuOffset = nextObuOffset; + } + + Assert.Equal(ObuType.Frame, (ObuType)((validPayload[finalObuOffset] >> 3) & 0x0F)); + Assert.Equal(1, finalSizeFieldLength); + Assert.InRange(finalPayloadLength, (ulong)(TruncatedTileByteCount + 1), 0x7FUL); + + byte[] truncatedPayload = validPayload[..^TruncatedTileByteCount]; + truncatedPayload[finalSizeFieldOffset] = (byte)(finalPayloadLength - TruncatedTileByteCount); + + using Av1Decoder decoder = new(Configuration.Default); + + Assert.Throws( + () => decoder.DecodeFrameBuffer(truncatedPayload, null, null, out _).Dispose()); + + Assert.Null(decoder.SequenceHeader); + Assert.Null(decoder.FrameHeader); + Assert.Null(decoder.FrameInfo); + } + /// /// Verifies decoded luma and chroma palette syntax and exact presented pixels for an independently encoded AVIF /// image across the available vector widths and the scalar fallback. @@ -222,6 +274,42 @@ public class Av1ReconstructionConformanceTests public void DecodeWithPaletteMatchesPinnedLibavifPresentation() => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidatePalettePresentedFixture, PresentationConfigurations); + /// + /// Verifies that malformed data following a decoded palette tile releases its frame state before the same decoder + /// processes another payload. + /// + [Fact] + public void DecodeFrameBufferRecoversAfterMalformedFollowingObu() + { + byte[] validPayload = TestFile.Create(TestImages.Heif.Av1Palette8BitPayload).Bytes; + + // The palette fixture ends with one combined-frame OBU containing one tile, so the intact prefix creates and + // completes a real Av1TileReader. The appended padding OBU declares one zero byte; AV1 padding requires a + // trailing-one bit, making this later bounded-payload failure deterministic without corrupting tile entropy. + byte[] malformedPayload = + [ + .. validPayload, + 0x7A, // Padding OBU with an explicit payload-size field. + 0x01, // LEB128 payload length of one byte. + 0x00, // Invalid padding payload with no trailing-one bit. + ]; + + using Av1Decoder decoder = new(Configuration.Default); + + Assert.Throws( + () => decoder.DecodeFrameBuffer(malformedPayload, null, null, out _).Dispose()); + + Assert.Null(decoder.SequenceHeader); + Assert.Null(decoder.FrameHeader); + Assert.Null(decoder.FrameInfo); + + using Av1FrameBuffer recoveredFrameBuffer = decoder.DecodeFrameBuffer(validPayload, null, null, out _); + + Assert.Equal(33, recoveredFrameBuffer.Width); + Assert.Equal(11, recoveredFrameBuffer.Height); + Assert.Equal(RequiredPaletteCoverage, GetPaletteCoverage(decoder)); + } + /// /// Verifies selected intra-block-copy prediction and exact native samples against scalar libaom for an /// independently encoded AV1 still pictures across every available vector width and the scalar fallback. diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceFrameStoreTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceFrameStoreTests.cs new file mode 100644 index 000000000..dddb14aaf --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceFrameStoreTests.cs @@ -0,0 +1,607 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain; +using SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies AV1 frame ownership, padded-buffer copying, and reference-border preservation. +/// +[Trait("Format", "Avif")] +[ValidateDisposedMemoryAllocations] +public class Av1ReferenceFrameStoreTests +{ + /// + /// Verifies that a zero refresh mask leaves the slot map and caller ownership unchanged. + /// + [Fact] + public void ApplyZeroRefreshMaskPreservesCallerOwnership() + { + using Av1ReferenceFrameStore store = new(); + using Av1ReferenceFrame frame = CreateFrame(); + + Assert.False(store.Commit(0, frame, showFrame: false)); + Assert.Null(store.Resolve(0)); + Assert.NotNull(frame.FrameBuffer.BufferY); + } + + /// + /// Verifies that every selected slot shares the one transferred frame owner. + /// + [Fact] + public void ApplyRefreshMaskStoresFrameInEverySelectedSlot() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame frame = CreateFrame(); + + Assert.True(store.Commit(0b1000_0101, frame, showFrame: false)); + Assert.Same(frame, store.Resolve(0)); + Assert.Null(store.Resolve(1)); + Assert.Same(frame, store.Resolve(2)); + Assert.Same(frame, store.Resolve(7)); + } + + /// + /// Verifies that a shown frame transfers to presentation ownership even when it refreshes no reference slot. + /// + [Fact] + public void ShownFrameWithZeroRefreshMaskTransfersOwnership() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame frame = CreateFrame(); + + Assert.True(store.Commit(0, frame, showFrame: true)); + Assert.Same(frame, store.OutputFrame); + Assert.Null(store.Resolve(0)); + } + + /// + /// Verifies that replacing the shown output does not release its predecessor while a reference slot retains it. + /// + [Fact] + public void ReplacingOutputPreservesReferencedPredecessor() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame firstOutput = CreateFrame(); + Av1ReferenceFrame secondOutput = CreateFrame(); + Av1ReferenceFrame replacementReference = CreateFrame(); + Av1FrameBuffer firstOutputBuffer = firstOutput.FrameBuffer; + store.Commit(0b0000_0001, firstOutput, showFrame: true); + + store.Commit(0, secondOutput, showFrame: true); + + Assert.NotNull(firstOutputBuffer.BufferY); + Assert.Same(firstOutput, store.Resolve(0)); + + store.Commit(0b0000_0001, replacementReference, showFrame: false); + + Assert.Null(firstOutputBuffer.BufferY); + } + + /// + /// Verifies that an independently committed presentation output does not release the ungrained owner retained by a reference slot. + /// + [Fact] + public void CommitOutputPreservesReferencedPreviousOutput() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame retainedReference = CreateFrame(); + Av1ReferenceFrame presentation = CreateFrame(); + Av1ReferenceFrame nextPresentation = CreateFrame(); + Av1ReferenceFrame replacementReference = CreateFrame(); + Av1FrameBuffer retainedReferenceBuffer = retainedReference.FrameBuffer; + Av1FrameBuffer presentationBuffer = presentation.FrameBuffer; + store.Commit(0b0000_0001, retainedReference, showFrame: true); + + store.CommitOutput(presentation); + + Assert.Same(presentation, store.OutputFrame); + Assert.Same(retainedReference, store.Resolve(0)); + Assert.NotNull(retainedReferenceBuffer.BufferY); + + store.CommitOutput(nextPresentation); + + Assert.Same(nextPresentation, store.OutputFrame); + Assert.Null(presentationBuffer.BufferY); + Assert.NotNull(retainedReferenceBuffer.BufferY); + + store.Commit(0b0000_0001, replacementReference, showFrame: false); + + Assert.Null(retainedReferenceBuffer.BufferY); + } + + /// + /// Verifies that replacing one alias does not release a frame retained by another slot. + /// + [Fact] + public void PartialReplacementPreservesSharedOwner() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame sharedFrame = CreateFrame(); + Av1ReferenceFrame replacement = CreateFrame(); + store.Commit(0b0000_0011, sharedFrame, showFrame: false); + + store.Commit(0b0000_0001, replacement, showFrame: false); + + Assert.Same(replacement, store.Resolve(0)); + Assert.Same(sharedFrame, store.Resolve(1)); + Assert.NotNull(sharedFrame.FrameBuffer.BufferY); + } + + /// + /// Verifies that replacing the final alias releases the displaced frame planes. + /// + [Fact] + public void FinalReplacementReleasesDisplacedOwner() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame displacedFrame = CreateFrame(); + Av1ReferenceFrame firstReplacement = CreateFrame(); + Av1ReferenceFrame secondReplacement = CreateFrame(); + Av1FrameBuffer displacedBuffer = displacedFrame.FrameBuffer; + store.Commit(0b0000_0011, displacedFrame, showFrame: false); + store.Commit(0b0000_0001, firstReplacement, showFrame: false); + + store.Commit(0b0000_0010, secondReplacement, showFrame: false); + + Assert.Null(displacedBuffer.BufferY); + Assert.Same(firstReplacement, store.Resolve(0)); + Assert.Same(secondReplacement, store.Resolve(1)); + } + + /// + /// Verifies that resetting the map releases one multiply referenced owner and clears every slot. + /// + [Fact] + public void ResetReleasesUniqueOwnersAndClearsSlots() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame frame = CreateFrame(); + Av1FrameBuffer frameBuffer = frame.FrameBuffer; + store.Commit(byte.MaxValue, frame, showFrame: false); + + store.Reset(); + + Assert.Null(frameBuffer.BufferY); + for (int slot = 0; slot < Av1Constants.ReferenceFrameCount; slot++) + { + Assert.Null(store.Resolve(slot)); + } + } + + /// + /// Verifies that taking the final output transfers its planes without copying and releases unrelated references. + /// + [Fact] + public void TakeOutputTransfersPlanesAndReleasesOtherReferences() + { + using Av1ReferenceFrameStore store = new(); + Av1ReferenceFrame reference = CreateFrame(); + Av1ReferenceFrame output = CreateFrame(); + Av1FrameBuffer referenceBuffer = reference.FrameBuffer; + store.Commit(0b0000_0001, reference, showFrame: false); + store.Commit(0b0000_0010, output, showFrame: true); + + using Av1ReferenceFrame selectedOutput = store.TakeOutput(); + using Av1FrameBuffer frameBuffer = selectedOutput.TakeFrameBuffer(); + + Assert.Same(output, selectedOutput); + Assert.Null(referenceBuffer.BufferY); + Assert.NotNull(frameBuffer.BufferY); + Assert.Null(store.OutputFrame); + Assert.Null(store.Resolve(0)); + Assert.Null(store.Resolve(1)); + } + + /// + /// Verifies that an eight-bit presentation copy contains every byte of each padded plane and the complete active geometry. + /// + [Fact] + public void CopyToCopiesCompletePaddedEightBitFrame() + => ValidateCompleteFrameCopy(Av1BitDepth.EightBit); + + /// + /// Verifies that a high-bit-depth presentation copy contains every native sample of each padded plane and the complete active geometry. + /// + [Fact] + public void CopyToCopiesCompletePaddedHighBitDepthFrame() + => ValidateCompleteFrameCopy(Av1BitDepth.TwelveBit); + + /// + /// Verifies that AV1 reference-border extension repeats the nearest visible edge across every allocated plane sample. + /// + /// The coded sample precision. + /// Whether the frame contains only luma. + /// Whether chroma is horizontally subsampled. + /// Whether chroma is vertically subsampled. + [Theory] + [InlineData(Av1BitDepth.EightBit, true, false, false)] + [InlineData(Av1BitDepth.EightBit, false, true, true)] + [InlineData(Av1BitDepth.TenBit, true, false, false)] + [InlineData(Av1BitDepth.TenBit, false, true, false)] + [InlineData(Av1BitDepth.TwelveBit, true, false, false)] + [InlineData(Av1BitDepth.TwelveBit, false, true, true)] + public void ExtendRepeatsVisibleEdgesAcrossCompletePadding( + int bitDepth, + bool isMonochrome, + bool subsamplingX, + bool subsamplingY) + { + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + 5, + 3, + (Av1BitDepth)bitDepth, + isMonochrome, + subsamplingX, + subsamplingY); + + using Av1FrameBuffer frameBuffer = new( + Configuration.Default, + sequenceHeader, + sequenceHeader.ColorConfig.GetColorFormat(), + false); + + InitializeVisiblePlane( + frameBuffer, + frameBuffer.BufferY!, + frameBuffer.OriginX, + frameBuffer.OriginY, + frameBuffer.Width, + frameBuffer.Height, + 0); + + if (!isMonochrome) + { + int subX = subsamplingX ? 1 : 0; + int subY = subsamplingY ? 1 : 0; + int chromaOriginX = frameBuffer.OriginX >> subX; + int chromaOriginY = frameBuffer.OriginY >> subY; + int chromaWidth = Av1Math.DivideLog2Ceiling(frameBuffer.Width, subX); + int chromaHeight = Av1Math.DivideLog2Ceiling(frameBuffer.Height, subY); + + InitializeVisiblePlane(frameBuffer, frameBuffer.BufferCb!, chromaOriginX, chromaOriginY, chromaWidth, chromaHeight, 1); + InitializeVisiblePlane(frameBuffer, frameBuffer.BufferCr!, chromaOriginX, chromaOriginY, chromaWidth, chromaHeight, 2); + } + + Av1ReferenceFrameBorder.Extend(frameBuffer); + + AssertExtendedPlane( + frameBuffer, + frameBuffer.BufferY!, + frameBuffer.OriginX, + frameBuffer.OriginY, + frameBuffer.Width, + frameBuffer.Height, + 0); + + if (!isMonochrome) + { + int subX = subsamplingX ? 1 : 0; + int subY = subsamplingY ? 1 : 0; + int chromaOriginX = frameBuffer.OriginX >> subX; + int chromaOriginY = frameBuffer.OriginY >> subY; + int chromaWidth = Av1Math.DivideLog2Ceiling(frameBuffer.Width, subX); + int chromaHeight = Av1Math.DivideLog2Ceiling(frameBuffer.Height, subY); + + AssertExtendedPlane(frameBuffer, frameBuffer.BufferCb!, chromaOriginX, chromaOriginY, chromaWidth, chromaHeight, 1); + AssertExtendedPlane(frameBuffer, frameBuffer.BufferCr!, chromaOriginX, chromaOriginY, chromaWidth, chromaHeight, 2); + } + } + + /// + /// Verifies that a refreshed shown frame retains its ungrained reconstruction while exposing an independently grained output. + /// + [Fact] + public void GrainedPresentationPreservesUngrainedReference() + { + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(8, 8, Av1BitDepth.EightBit, true, false, false); + ObuFrameHeader frameHeader = new() + { + ShowFrame = true, + RefreshFrameFlags = 1, + FilmGrainParameters = new ObuFilmGrainParameters + { + ApplyGrain = true, + GrainSeed = 7391, + NumYPoints = 2, + GrainScalingMinus8 = 0, + ArCoeffLag = 0, + ArCoeffShiftMinus6 = 0, + GrainScaleShift = 0 + } + }; + + frameHeader.FilmGrainParameters.PointYValue[0] = 0; + frameHeader.FilmGrainParameters.PointYValue[1] = 255; + frameHeader.FilmGrainParameters.PointYScaling[0] = 255; + frameHeader.FilmGrainParameters.PointYScaling[1] = 255; + + Av1FrameBuffer reconstructed = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + InitializeVisiblePlane( + reconstructed, + reconstructed.BufferY!, + reconstructed.OriginX, + reconstructed.OriginY, + reconstructed.Width, + reconstructed.Height, + 0); + + Av1ReferenceFrameBorder.Extend(reconstructed); + Span reconstructedSamples = reconstructed.BufferY!.DangerousGetSingleSpan(); + byte[] ungrainedSamples = new byte[reconstructedSamples.Length]; + reconstructedSamples.CopyTo(ungrainedSamples); + Av1FrameBuffer presentation = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + reconstructed.CopyTo(presentation); + + Av1FilmGrainDecoder filmGrainDecoder = new(sequenceHeader, frameHeader, presentation); + filmGrainDecoder.DecodeFrame(); + + Av1FrameInfo frameInfo = new(sequenceHeader); + Av1ReferenceFrame retainedReference = new(reconstructed, frameHeader, frameInfo); + Av1ReferenceFrame grainedOutput = new(presentation, frameHeader, frameInfo); + Av1FrameBuffer retainedReferenceBuffer = retainedReference.FrameBuffer; + using Av1ReferenceFrameStore store = new(); + store.Commit(frameHeader.RefreshFrameFlags, retainedReference, showFrame: false); + store.CommitOutput(grainedOutput); + + Assert.Same(retainedReference, store.Resolve(0)); + Assert.Same(grainedOutput, store.OutputFrame); + Assert.NotSame(retainedReference.FrameBuffer, grainedOutput.FrameBuffer); + Assert.True(ungrainedSamples.AsSpan().SequenceEqual(retainedReference.FrameBuffer.BufferY!.DangerousGetSingleSpan())); + Assert.False(ungrainedSamples.AsSpan().SequenceEqual(grainedOutput.FrameBuffer.BufferY!.DangerousGetSingleSpan())); + + using Av1ReferenceFrame selectedOutput = store.TakeOutput(); + + Assert.Same(grainedOutput, selectedOutput); + Assert.Null(retainedReferenceBuffer.BufferY); + Assert.NotNull(selectedOutput.FrameBuffer.BufferY); + Assert.Null(store.OutputFrame); + Assert.Null(store.Resolve(0)); + } + + /// + /// Verifies a complete padded-plane copy for one native AV1 sample precision. + /// + /// The coded sample precision. + private static void ValidateCompleteFrameCopy(Av1BitDepth bitDepth) + { + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(5, 3, bitDepth, false, true, true); + using Av1FrameBuffer source = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false); + using Av1FrameBuffer destination = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv420, false); + + source.OriginX--; + source.OriginY--; + source.Width = 4; + source.Height = 2; + source.MaxWidth = 4; + source.MaxHeight = 2; + FillCompletePlane(source, source.BufferY!, 17); + FillCompletePlane(source, source.BufferCb!, 53); + FillCompletePlane(source, source.BufferCr!, 89); + destination.BufferY!.DangerousGetSingleSpan().Fill(0xA5); + destination.BufferCb!.DangerousGetSingleSpan().Fill(0xA5); + destination.BufferCr!.DangerousGetSingleSpan().Fill(0xA5); + + source.CopyTo(destination); + + Assert.True(source.BufferY!.DangerousGetSingleSpan().SequenceEqual(destination.BufferY!.DangerousGetSingleSpan())); + Assert.True(source.BufferCb!.DangerousGetSingleSpan().SequenceEqual(destination.BufferCb!.DangerousGetSingleSpan())); + Assert.True(source.BufferCr!.DangerousGetSingleSpan().SequenceEqual(destination.BufferCr!.DangerousGetSingleSpan())); + Assert.Equal(source.StartPosition, destination.StartPosition); + Assert.Equal(source.OriginX, destination.OriginX); + Assert.Equal(source.OriginY, destination.OriginY); + Assert.Equal(source.Width, destination.Width); + Assert.Equal(source.Height, destination.Height); + Assert.Equal(source.MaxWidth, destination.MaxWidth); + Assert.Equal(source.MaxHeight, destination.MaxHeight); + Assert.Equal(source.BitDepth, destination.BitDepth); + Assert.Equal(source.ColorFormat, destination.ColorFormat); + + byte sourceFirstSample = source.BufferY!.DangerousGetSingleSpan()[0]; + + destination.BufferY!.DangerousGetSingleSpan()[0] ^= byte.MaxValue; + Assert.Equal(sourceFirstSample, source.BufferY!.DangerousGetSingleSpan()[0]); + } + + /// + /// Fills every storage element of one padded plane with deterministic native sample data. + /// + /// The frame that defines the native sample size. + /// The complete padded plane. + /// The plane-specific value mixed into each sample. + private static void FillCompletePlane(Av1FrameBuffer frameBuffer, Buffer2D buffer, int seed) + { + if (frameBuffer.BytesPerSample == 1) + { + Span samples = buffer.DangerousGetSingleSpan(); + + for (int i = 0; i < samples.Length; i++) + { + samples[i] = (byte)((seed + (i * 17)) & byte.MaxValue); + } + + return; + } + + Span highBitDepthSamples = MemoryMarshal.Cast(buffer.DangerousGetSingleSpan()); + + for (int i = 0; i < highBitDepthSamples.Length; i++) + { + highBitDepthSamples[i] = (ushort)((seed + (i * 29)) & 0xFFF); + } + } + + /// + /// Initializes one visible plane with unique samples while leaving a distinct sentinel throughout its padding. + /// + /// The frame that defines the native sample size. + /// The padded plane to initialize. + /// The horizontal visible origin in plane samples. + /// The vertical visible origin in rows. + /// The visible plane width. + /// The visible plane height. + /// The zero-based plane index mixed into the visible samples. + private static void InitializeVisiblePlane( + Av1FrameBuffer frameBuffer, + Buffer2D buffer, + int originX, + int originY, + int width, + int height, + int planeIndex) + { + if (frameBuffer.BytesPerSample == 1) + { + Span samples = buffer.DangerousGetSingleSpan(); + samples.Fill(byte.MaxValue); + int stride = buffer.Width; + + for (int row = 0; row < height; row++) + { + for (int column = 0; column < width; column++) + { + samples[((originY + row) * stride) + originX + column] = (byte)GetVisibleSample(planeIndex, row, column); + } + } + + return; + } + + Span highBitDepthSamples = MemoryMarshal.Cast(buffer.DangerousGetSingleSpan()); + highBitDepthSamples.Fill(ushort.MaxValue); + int highBitDepthStride = buffer.Width >> 1; + + for (int row = 0; row < height; row++) + { + for (int column = 0; column < width; column++) + { + highBitDepthSamples[((originY + row) * highBitDepthStride) + originX + column] = + (ushort)GetVisibleSample(planeIndex, row, column); + } + } + } + + /// + /// Verifies every sample in one padded plane against nearest-edge replication of the initialized visible rectangle. + /// + /// The frame that defines the native sample size. + /// The padded plane to verify. + /// The horizontal visible origin in plane samples. + /// The vertical visible origin in rows. + /// The visible plane width. + /// The visible plane height. + /// The zero-based plane index mixed into the visible samples. + private static void AssertExtendedPlane( + Av1FrameBuffer frameBuffer, + Buffer2D buffer, + int originX, + int originY, + int width, + int height, + int planeIndex) + { + int stride = buffer.Width / frameBuffer.BytesPerSample; + int allocatedHeight = buffer.Height; + + if (frameBuffer.BytesPerSample == 1) + { + ReadOnlySpan samples = buffer.DangerousGetSingleSpan(); + + for (int row = 0; row < allocatedHeight; row++) + { + int visibleRow = Math.Clamp(row - originY, 0, height - 1); + + for (int column = 0; column < stride; column++) + { + int visibleColumn = Math.Clamp(column - originX, 0, width - 1); + byte expected = (byte)GetVisibleSample(planeIndex, visibleRow, visibleColumn); + byte actual = samples[(row * stride) + column]; + + if (actual != expected) + { + Assert.Equal(expected, actual); + } + } + } + + return; + } + + ReadOnlySpan highBitDepthSamples = MemoryMarshal.Cast(buffer.DangerousGetSingleSpan()); + + for (int row = 0; row < allocatedHeight; row++) + { + int visibleRow = Math.Clamp(row - originY, 0, height - 1); + + for (int column = 0; column < stride; column++) + { + int visibleColumn = Math.Clamp(column - originX, 0, width - 1); + ushort expected = (ushort)GetVisibleSample(planeIndex, visibleRow, visibleColumn); + ushort actual = highBitDepthSamples[(row * stride) + column]; + + if (actual != expected) + { + Assert.Equal(expected, actual); + } + } + } + } + + /// + /// Computes the deterministic visible sample used by the border-extension oracle. + /// + /// The zero-based plane index. + /// The visible row. + /// The visible column. + /// The native sample value. + private static int GetVisibleSample(int planeIndex, int row, int column) => ((planeIndex + 1) * 31) + (row * 11) + (column * 3); + + /// + /// Creates the smallest valid monochrome reference-frame owner for slot-lifecycle tests. + /// + /// A reference frame whose sample planes are owned by the caller. + private static Av1ReferenceFrame CreateFrame() + { + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(1, 1, Av1BitDepth.EightBit, true, false, false); + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + + return new Av1ReferenceFrame(frameBuffer, new ObuFrameHeader(), new Av1FrameInfo(sequenceHeader)); + } + + /// + /// Creates the sequence geometry and color configuration used by direct frame-buffer tests. + /// + /// The maximum coded width. + /// The maximum coded height. + /// The coded sample precision. + /// Whether the sequence contains only luma. + /// Whether chroma is horizontally subsampled. + /// Whether chroma is vertically subsampled. + /// The initialized sequence header. + private static ObuSequenceHeader CreateSequenceHeader( + int width, + int height, + Av1BitDepth bitDepth, + bool isMonochrome, + bool subsamplingX, + bool subsamplingY) + => new() + { + MaxFrameWidth = width, + MaxFrameHeight = height, + ColorConfig = new ObuColorConfig + { + IsMonochrome = isMonochrome, + SubSamplingX = subsamplingX, + SubSamplingY = subsamplingY, + BitDepth = bitDepth + } + }; +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs index a2afda2ac..174188a8c 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs @@ -12,6 +12,11 @@ internal class Av1TileDecoderStub : IAv1TileReader, IAv1TileWriter public void ReadTile(Span tileData, int tileNum) => this.tileDatas.Add(tileNum, tileData.ToArray()); + /// + public void CompleteFrame() + { + } + public Span WriteTile(int tileNum) => this.tileDatas[tileNum]; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameLifecycleTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameLifecycleTests.cs new file mode 100644 index 000000000..699066d70 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameLifecycleTests.cs @@ -0,0 +1,402 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.ReferenceFrames; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// Verifies frame ownership and completion while parsing layered AV1 OBU payloads. +/// +[Trait("Format", "Avif")] +public class ObuFrameLifecycleTests +{ + private const int ProgressiveSequencePrefixLength = 19; + private const int FirstProgressiveLayerLength = 55; + private const int NoFailingReaderIndex = -1; + private const int SecondFrameReaderIndex = 1; + private const byte ForbiddenObuHeader = 0x80; + private const byte InvalidTrailingByte = 1; + + /// + /// The operating point in the fixture sequence header that selects both spatial layers. + /// + private const byte ProgressiveOperatingPointIndex = 0; + + /// + /// The source and displayed width recorded by libavif for both progressive layers. + /// + private const int ProgressiveImageWidth = 33; + + /// + /// The source and displayed height recorded by libavif for both progressive layers. + /// + private const int ProgressiveImageHeight = 11; + + // This is the complete 72-byte color-item payload assembled from both extents of libavif's + // draw_points_idat_progressive.avif. It contains one sequence header followed by two coded + // spatial layers, so the lifecycle test exercises real progressive item framing. + private static ReadOnlySpan ProgressiveTwoFrameObuStream => + [ + // Temporal delimiter and progressive sequence header. + 0x12, 0x00, + 0x0A, 0x0F, 0x20, 0x13, 0x01, 0x00, 0x80, 0x81, 0x4E, 0x0A, 0x36, 0xBE, 0x48, 0x08, 0x20, 0x34, 0x80, + + // Base spatial layer: unextended combined-frame OBU with a 34-byte payload. + 0x32, 0x22, 0x14, 0x00, 0x27, 0xC0, 0x00, 0x00, 0x80, 0x00, 0x20, 0xF4, 0xAC, 0x60, 0x4B, 0x59, + 0xB6, 0x97, 0xB3, 0xD1, 0xF9, 0x22, 0xB7, 0x5B, 0xAC, 0xD5, 0xD1, 0x99, 0x8C, 0x5F, 0x30, 0x67, + 0xB4, 0x6C, 0x99, 0x80, + + // Enhancement spatial layer: extended combined-frame OBU with temporal_id 0 and spatial_id 1. + 0x36, 0x08, 0x0E, 0x33, 0x01, 0xC0, 0x20, 0x00, 0x00, 0x06, 0x80, 0x01, 0x00, 0xF3, 0xA2, 0xD4, 0x38 + ]; + + /// + /// Verifies that a bounded progressive payload is consumed completely and assigns fresh tile state to each coded frame. + /// + [Fact] + public void ReadAllConsumesBothProgressiveFrames() + { + const int temporalId = 0; + const int baseSpatialId = 0; + const int enhancementSpatialId = 1; + byte[] bitStream = [.. ProgressiveTwoFrameObuStream]; + Av1BitStreamReader reader = new(bitStream); + using Av1ReferenceFrameStore referenceFrames = new(); + ObuReader obuReader = new(ProgressiveOperatingPointIndex, referenceFrames); + LifecycleTileReaderFactory factory = new(obuReader, referenceFrames, NoFailingReaderIndex); + + obuReader.ReadAll(ref reader, bitStream.Length, factory.Create); + + ObuFrameHeader secondFrameHeader = Assert.IsType(factory.Readers[1].CompletedFrameHeader); + + Assert.Equal(bitStream.Length * 8, reader.BitPosition); + Assert.Equal(2, factory.Readers.Count); + Assert.NotSame(factory.Readers[0], factory.Readers[1]); + Assert.All(factory.Readers, frameReader => Assert.Equal(1, frameReader.TileCount)); + Assert.All(factory.Readers, frameReader => Assert.Equal(1, frameReader.CompletionCount)); + Assert.All(factory.Readers, frameReader => Assert.Equal(1, frameReader.TileCountAtCompletion)); + Assert.Equal(temporalId, factory.Readers[0].CompletedTemporalId); + Assert.Equal(baseSpatialId, factory.Readers[0].CompletedSpatialId); + Assert.Equal(temporalId, factory.Readers[1].CompletedTemporalId); + Assert.Equal(enhancementSpatialId, factory.Readers[1].CompletedSpatialId); + Assert.True(factory.Readers[1].SelectedReferencesWereRetained); + Assert.Equal(ObuFrameType.InterFrame, secondFrameHeader.FrameType); + Assert.Equal(ProgressiveImageWidth, secondFrameHeader.FrameSize.SuperResolutionUpscaledWidth); + Assert.Equal(ProgressiveImageHeight, secondFrameHeader.FrameSize.FrameHeight); + Assert.Equal(ProgressiveImageWidth, secondFrameHeader.FrameSize.RenderWidth); + Assert.Equal(ProgressiveImageHeight, secondFrameHeader.FrameSize.RenderHeight); + } + + /// + /// Verifies that invalid trailing data in a subsequent sequence header is rejected before that sequence can create frame state. + /// + [Fact] + public void ReadAllRejectsMalformedSubsequentSequenceBeforeCreatingItsFrameReader() + { + byte[] firstLayer = ProgressiveTwoFrameObuStream[..FirstProgressiveLayerLength].ToArray(); + byte[] malformedSequence = AddInvalidSequenceHeaderTrailingByte(ProgressiveTwoFrameObuStream[..ProgressiveSequencePrefixLength].ToArray()); + byte[] bitStream = [.. firstLayer, .. malformedSequence]; + using Av1ReferenceFrameStore referenceFrames = new(); + ObuReader obuReader = new(ProgressiveOperatingPointIndex, referenceFrames); + LifecycleTileReaderFactory factory = new(obuReader, referenceFrames, NoFailingReaderIndex); + + Assert.Throws(() => ReadObuStream(bitStream, obuReader, factory.Create)); + AssertParserSessionReset(obuReader, referenceFrames); + LifecycleTileReader completedFrame = Assert.Single(factory.Readers); + + Assert.Equal(1, completedFrame.TileCount); + Assert.Equal(1, completedFrame.CompletionCount); + + ReadObuStream(ProgressiveTwoFrameObuStream.ToArray(), obuReader, factory.Create); + + Assert.Equal(3, factory.Readers.Count); + Assert.Equal(1, factory.Readers[1].CompletionCount); + Assert.Equal(1, factory.Readers[2].CompletionCount); + } + + /// + /// Verifies that synchronous tile validation failure prevents completion of the affected coded frame. + /// + [Fact] + public void ReadAllDoesNotCompleteFrameWhenTileValidationFails() + { + byte[] bitStream = [.. ProgressiveTwoFrameObuStream]; + using Av1ReferenceFrameStore referenceFrames = new(); + ObuReader obuReader = new(ProgressiveOperatingPointIndex, referenceFrames); + LifecycleTileReaderFactory factory = new(obuReader, referenceFrames, SecondFrameReaderIndex); + + Assert.Throws(() => ReadObuStream(bitStream, obuReader, factory.Create)); + Assert.Equal(2, factory.Readers.Count); + Assert.Equal(1, factory.Readers[0].CompletionCount); + Assert.Equal(1, factory.Readers[1].TileCount); + Assert.Equal(0, factory.Readers[1].CompletionCount); + AssertParserSessionReset(obuReader, referenceFrames); + + ReadObuStream(bitStream, obuReader, factory.Create); + + Assert.Equal(4, factory.Readers.Count); + Assert.Equal(1, factory.Readers[2].CompletionCount); + Assert.Equal(1, factory.Readers[3].CompletionCount); + } + + /// + /// Verifies that malformed data after a completed real frame invalidates retained state without preventing reuse of the parser. + /// + [Fact] + public void ReadAllClearsCompletedFrameStateWhenFollowingObuHeaderIsInvalid() + { + byte[] bitStream = [.. ProgressiveTwoFrameObuStream[..FirstProgressiveLayerLength], ForbiddenObuHeader]; + using Av1ReferenceFrameStore referenceFrames = new(); + ObuReader obuReader = new(ProgressiveOperatingPointIndex, referenceFrames); + LifecycleTileReaderFactory factory = new(obuReader, referenceFrames, NoFailingReaderIndex); + + Assert.Throws(() => ReadObuStream(bitStream, obuReader, factory.Create)); + AssertParserSessionReset(obuReader, referenceFrames); + Assert.Equal(1, Assert.Single(factory.Readers).CompletionCount); + + ReadObuStream(ProgressiveTwoFrameObuStream.ToArray(), obuReader, factory.Create); + + Assert.Equal(3, factory.Readers.Count); + Assert.Equal(1, factory.Readers[1].CompletionCount); + Assert.Equal(1, factory.Readers[2].CompletionCount); + } + + /// + /// Extends the sequence-header OBU by one nonzero byte while retaining all following encoded frame bytes. + /// + /// A temporal-delimiter and sequence-header OBU prefix. + /// A stream whose sequence-header syntax has invalid nonzero trailing data. + private static byte[] AddInvalidSequenceHeaderTrailingByte(byte[] stream) + { + int sequenceObuOffset = GetNextObuOffset(stream, 0); + Av1BitStreamReader sizeReader = new(stream.AsSpan(sequenceObuOffset + 1)); + uint sequencePayloadLength = (uint)sizeReader.ReadLittleEndianBytes128(out int oldSizeLength); + int sequencePayloadOffset = sequenceObuOffset + 1 + oldSizeLength; + int frameObuOffset = sequencePayloadOffset + (int)sequencePayloadLength; + Span encodedSize = stackalloc byte[5]; + int newSizeLength = Av1BitStreamWriter.GetLittleEndianBytes128(sequencePayloadLength + 1, encodedSize); + byte[] malformed = new byte[stream.Length + 1 + newSizeLength - oldSizeLength]; + + stream.AsSpan(0, sequenceObuOffset + 1).CopyTo(malformed); + encodedSize[..newSizeLength].CopyTo(malformed.AsSpan(sequenceObuOffset + 1)); + stream.AsSpan(sequencePayloadOffset, (int)sequencePayloadLength) + .CopyTo(malformed.AsSpan(sequenceObuOffset + 1 + newSizeLength)); + + int trailingByteOffset = sequenceObuOffset + 1 + newSizeLength + (int)sequencePayloadLength; + malformed[trailingByteOffset] = InvalidTrailingByte; + stream.AsSpan(frameObuOffset).CopyTo(malformed.AsSpan(trailingByteOffset + 1)); + return malformed; + } + + /// + /// Gets the byte offset immediately following one explicitly sized OBU. + /// + /// The complete OBU stream. + /// The fixed-header offset of the current OBU. + /// The fixed-header offset of the following OBU. + private static int GetNextObuOffset(byte[] stream, int obuOffset) + { + Av1BitStreamReader sizeReader = new(stream.AsSpan(obuOffset + 1)); + ulong payloadLength = sizeReader.ReadLittleEndianBytes128(out int sizeLength); + return obuOffset + 1 + sizeLength + (int)payloadLength; + } + + /// + /// Reads a complete OBU stream through reference-type state so malformed-input assertions do not capture a ref struct. + /// + /// The complete bounded OBU stream. + /// The stateful OBU parser. + /// Creates the tile reader for each coded frame. + private static void ReadObuStream(byte[] stream, ObuReader obuReader, Func creator) + { + Av1BitStreamReader reader = new(stream); + obuReader.ReadAll(ref reader, stream.Length, creator); + } + + /// + /// Verifies that an unsuccessful bounded parse removed all state that could refer to the rejected session. + /// + /// The parser whose published header state must be empty. + /// The reference map whose retained frame owners must be empty. + private static void AssertParserSessionReset(ObuReader obuReader, Av1ReferenceFrameStore referenceFrames) + { + Assert.Null(obuReader.SequenceHeader); + Assert.Null(obuReader.FrameHeader); + + for (int slot = 0; slot < Av1Constants.ReferenceFrameCount; slot++) + { + Assert.Null(referenceFrames.Resolve(slot)); + } + } + + /// + /// Creates and retains one recording tile reader for every coded frame requested by the OBU parser. + /// + private sealed class LifecycleTileReaderFactory + { + private readonly ObuReader obuReader; + private readonly Av1ReferenceFrameStore referenceFrames; + private readonly int failingReaderIndex; + + /// + /// Initializes a new instance of the class. + /// + /// The parser that owns the current frame-header state. + /// The reconstructed reference map shared with the parser. + /// The zero-based reader index whose tile validation should fail. + public LifecycleTileReaderFactory( + ObuReader obuReader, + Av1ReferenceFrameStore referenceFrames, + int failingReaderIndex) + { + this.obuReader = obuReader; + this.referenceFrames = referenceFrames; + this.failingReaderIndex = failingReaderIndex; + } + + /// + /// Gets the tile readers created in coded-frame order. + /// + public List Readers { get; } = []; + + /// + /// Creates a fresh recording tile reader for the parser's current coded frame. + /// + /// The fresh tile reader. + public LifecycleTileReader Create() + { + LifecycleTileReader reader = new( + this.obuReader, + this.referenceFrames, + this.Readers.Count == this.failingReaderIndex); + + this.Readers.Add(reader); + return reader; + } + } + + /// + /// Records tile-reader ownership and the frame-header state observable at completion. + /// + private sealed class LifecycleTileReader : IAv1TileReader + { + private readonly ObuReader obuReader; + private readonly Av1ReferenceFrameStore referenceFrames; + private readonly bool failTileValidation; + + /// + /// Initializes a new instance of the class. + /// + /// The parser whose current frame header is captured at completion. + /// The reconstructed reference map shared with the parser. + /// A value indicating whether tile validation should fail. + public LifecycleTileReader( + ObuReader obuReader, + Av1ReferenceFrameStore referenceFrames, + bool failTileValidation) + { + this.obuReader = obuReader; + this.referenceFrames = referenceFrames; + this.failTileValidation = failTileValidation; + } + + /// + /// Gets the number of tile payloads delivered to this frame reader. + /// + public int TileCount { get; private set; } + + /// + /// Gets the number of frame-completion notifications delivered to this frame reader. + /// + public int CompletionCount { get; private set; } + + /// + /// Gets the number of delivered tile payloads observed when the frame was completed. + /// + public int TileCountAtCompletion { get; private set; } + + /// + /// Gets the primary frame-header OBU temporal identifier observed at completion. + /// + public int CompletedTemporalId { get; private set; } + + /// + /// Gets the primary frame-header OBU spatial identifier observed at completion. + /// + public int CompletedSpatialId { get; private set; } + + /// + /// Gets the frame header observed at successful completion. + /// + public ObuFrameHeader CompletedFrameHeader { get; private set; } + + /// + /// Gets a value indicating whether every inter-reference role resolved to a retained reconstructed owner before + /// the completed frame changed the reference map. + /// + public bool SelectedReferencesWereRetained { get; private set; } + + /// + public void ReadTile(Span tileData, int tileNum) + { + this.TileCount++; + if (this.failTileValidation) + { + // The final tile owns the remaining declared OBU payload. A trailing-symbol or entropy validation + // failure therefore originates at this boundary and must prevent the later completion callback. + throw new InvalidImageContentException("The test tile payload failed validation."); + } + } + + /// + public void CompleteFrame() + { + ObuSequenceHeader sequenceHeader = Assert.IsType(this.obuReader.SequenceHeader); + ObuFrameHeader frameHeader = Assert.IsType(this.obuReader.FrameHeader); + bool selectedReferencesWereRetained = true; + + if (!frameHeader.IsIntra) + { + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + + // Resolve all seven roles before committing the current frame. The refresh mask may replace those + // slots, so checking after Commit would verify the new owner instead of the references just parsed. + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + selectedReferencesWereRetained &= this.referenceFrames.Resolve((int)referenceFrameIndices[reference]) is not null; + } + } + + Av1FrameInfo frameInfo = new(sequenceHeader); + Av1FrameBuffer frameBuffer = new( + Configuration.Default, + sequenceHeader, + sequenceHeader.ColorConfig.GetColorFormat(), + is16BitPipeline: false); + + Av1ReferenceFrame referenceFrame = new(frameBuffer, frameHeader, frameInfo); + + // A retained AV1 buffer exposes the visible post-super-resolution geometry rather than the sequence maxima + // used for allocation. frame_size_with_refs reads these exact dimensions for the following coded layer. + frameBuffer.Width = frameHeader.FrameSize.SuperResolutionUpscaledWidth; + frameBuffer.Height = frameHeader.FrameSize.FrameHeight; + + if (!this.referenceFrames.Commit(frameHeader.RefreshFrameFlags, referenceFrame, showFrame: frameHeader.ShowFrame)) + { + // A hidden frame with no refresh role remains caller-owned; match production by releasing it immediately. + referenceFrame.Dispose(); + } + + this.TileCountAtCompletion = this.TileCount; + this.CompletedTemporalId = frameHeader.TemporalId; + this.CompletedSpatialId = frameHeader.SpatialId; + this.CompletedFrameHeader = frameHeader; + this.SelectedReferencesWereRetained = selectedReferencesWereRetained; + this.CompletionCount++; + } + } +}