From 53204b223b961eb8240b4ce598ea7a0fda7fc94a Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Thu, 27 Aug 2026 00:04:46 +1000 Subject: [PATCH] Implement SIMD-first AV1 transform architecture --- HEIF_IMPLEMENTATION_PLAN.md | 152 +- .../Av1ByteInverseTransformOutputOperator.cs | 18 - .../Av1/Transform/Av1ForwardTransformer.cs | 964 ++++--- ...hBitDepthInverseTransformOutputOperator.cs | 14 - .../Av1/Transform/Av1Inverse2dTransformer.cs | 278 +- .../Heif/Av1/Transform/Av1Transform1dMath.cs | 209 ++ .../Av1/Transform/Av1Transform2dOperations.cs | 748 ++++++ .../Heif/Av1/Transform/Av1TransformVector.cs | 86 +- .../Av1/Transform/Av1TransformWorkspace.cs | 21 +- .../Av1Adst16Forward1dOperator.Simd.cs | 557 ---- .../Forward/Av1Adst16Forward1dOperator.cs | 190 +- .../Forward/Av1Adst4Forward1dOperator.Simd.cs | 113 - .../Forward/Av1Adst4Forward1dOperator.cs | 66 +- .../Forward/Av1Adst8Forward1dOperator.Simd.cs | 278 -- .../Forward/Av1Adst8Forward1dOperator.cs | 95 +- .../Forward/Av1Dct16Forward1dOperator.Simd.cs | 446 ---- .../Forward/Av1Dct16Forward1dOperator.cs | 153 +- .../Forward/Av1Dct32Forward1dOperator.Simd.cs | 986 ------- .../Forward/Av1Dct32Forward1dOperator.cs | 333 +-- .../Forward/Av1Dct4Forward1dOperator.Simd.cs | 119 - .../Forward/Av1Dct4Forward1dOperator.cs | 41 +- .../Forward/Av1Dct64Forward1dOperator.Simd.cs | 2246 ---------------- .../Forward/Av1Dct64Forward1dOperator.cs | 753 +----- .../Forward/Av1Dct8Forward1dOperator.Simd.cs | 218 -- .../Forward/Av1Dct8Forward1dOperator.cs | 75 +- .../Forward/Av1ForwardTransformArithmetic.cs | 848 ++++++ .../Av1ForwardTransformOperations.Adst.cs | 311 +++ .../Av1ForwardTransformOperations.Dct16.cs | 120 + .../Av1ForwardTransformOperations.Dct32.cs | 224 ++ .../Av1ForwardTransformOperations.Dct4.cs | 41 + .../Av1ForwardTransformOperations.Dct64.cs | 437 ++++ .../Av1ForwardTransformOperations.Dct8.cs | 66 + .../Av1ForwardTransformOperations.Identity.cs | 111 + .../Forward/Av1Identity16Forward1dOperator.cs | 32 +- .../Forward/Av1Identity32Forward1dOperator.cs | 32 +- .../Forward/Av1Identity4Forward1dOperator.cs | 32 +- .../Forward/Av1Identity8Forward1dOperator.cs | 32 +- .../Av1IdentityForward1dOperators.Simd.cs | 198 -- .../Transform/Forward/Av1TransformRounding.cs | 42 + .../Forward/IAv1ForwardTransform1dOperator.cs | 30 + .../IAv1InverseTransformOutputOperator.cs | 9 - .../Av1/Transform/IAv1Transform1dOperator.cs | 15 - .../Av1Adst16Inverse1dOperator.Simd.cs | 569 ----- .../Inverse/Av1Adst16Inverse1dOperator.cs | 371 +++ .../Inverse/Av1Adst4Inverse1dOperator.Simd.cs | 113 - .../Inverse/Av1Adst4Inverse1dOperator.cs | 70 + .../Inverse/Av1Adst8Inverse1dOperator.Simd.cs | 290 --- .../Inverse/Av1Adst8Inverse1dOperator.cs | 185 ++ .../Inverse/Av1Dct16Inverse1dOperator.Simd.cs | 476 ---- .../Inverse/Av1Dct16Inverse1dOperator.cs | 309 +++ .../Inverse/Av1Dct32Inverse1dOperator.Simd.cs | 1028 -------- .../Inverse/Av1Dct32Inverse1dOperator.cs | 677 +++++ .../Inverse/Av1Dct4Inverse1dOperator.Simd.cs | 113 - .../Inverse/Av1Dct4Inverse1dOperator.cs | 67 + .../Inverse/Av1Dct64Inverse1dOperator.Simd.cs | 2273 ----------------- .../Inverse/Av1Dct64Inverse1dOperator.cs | 1507 +++++++++++ .../Inverse/Av1Dct8Inverse1dOperator.Simd.cs | 233 -- .../Inverse/Av1Dct8Inverse1dOperator.cs | 147 ++ .../Inverse/Av1Identity16Inverse1dOperator.cs | 30 + .../Inverse/Av1Identity32Inverse1dOperator.cs | 30 + .../Inverse/Av1Identity4Inverse1dOperator.cs | 30 + .../Inverse/Av1Identity8Inverse1dOperator.cs | 30 + .../Av1IdentityInverse1dOperators.Simd.cs | 198 -- .../Codecs/Heif/Av1TransformBenchmarks.cs | 341 +-- .../Heif/Av1/Av1ForwardTransformTests.cs | 546 ++-- .../Heif/Av1/Av1InverseTransformTests.cs | 108 +- 66 files changed, 7955 insertions(+), 13525 deletions(-) delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformArithmetic.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1TransformRounding.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 29def16cc..baf930e6a 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -27,9 +27,48 @@ Full completion includes: Checkboxes may be marked complete only when the implementation and the verification required by the corresponding phase exit gate are both recorded. Source presence, compilation, self-roundtrip, or a temporary vertical slice is not completion evidence. -## Active and queued work +## Delivery dashboard -- [ ] **Active:** complete HEVC still-picture reconstruction and connect its native planes to the verified shared SIMD-first HEIF color pipeline. +Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `2bd81bb7c`, including the completed AV1 transform architecture checkpoint. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. + +Status meanings: + +- **Complete:** the implementation and its phase exit evidence are recorded. +- **In progress:** usable implementation exists, but one or more required behaviors or verification gates remain open. +- **Not started:** supporting primitives may exist, but the production format path is absent. +- **Current:** the only work item that should be advanced before taking the next queued item. + +Current development stage: **Stage 3 — complete AV1 still-image decoding.** The transform checkpoint is closed: forward transforms use one libaom-shaped operator architecture across scalar, `Vector128`, `Vector256`, and `Vector512`, inverse production traversal uses the verified scalar, `Vector128`, and `Vector256` tiers, and implementation-mechanic type and file suffixes have been removed. Neither AV1 nor HEVC production encoding is implemented. + +Immediate checkpoint: **remove every remaining valid AV1 still-image unsupported branch and prove the complete decode matrix.** Each syntax tool must be implemented through the established SIMD-first architecture with scalar fallback and verified with independent AVIF/libaom evidence across supported bit depths, chroma layouts, filters, grain, and color signaling. + +| Order | Delivery stage | State | Delivered state | Gate that remains open | +| --- | --- | --- | --- | --- | +| 1 | Baseline, provenance, documentation, and public contract | In progress | Pinned codec references, a bounded image-only scope, encoder options, typed bit depth, decoder-option propagation, and extensive HEIF documentation exist. | Complete the all-file documentation audit, record a fresh Release baseline, finish distinct public HEIC/AVIF save boundaries, and close API review. | +| 2 | Bounded HEIF item and image-sequence container | In progress | Still-item parsing, grids, auxiliary alpha, metadata properties, bounded image-sequence tracks, Identify, and all-sync AV1 sequence presentation are connected. | Complete adversarial boundary coverage, remaining item/property behavior, reference-dependent sequence reconstruction, and the bounded sequence writer. | +| 3 | Still-image AV1 and HEVC decoding | **Current** | HEVC reconstruction reaches exact HM/libheif fixtures across the recorded 8/10/12-bit and chroma cases. AV1 includes reconstruction, filter, grain, color, and SIMD-first transforms using one static-generic stage architecture across the supported vector widths. | Remove every valid AV1 still-image unsupported branch with independent vectors, followed by the remaining HEVC profile and Range Extensions conformance matrix. | +| 4 | Complete decoded presentation and animation | In progress | Shared SIMD-first AV1/HEVC color conversion, ICC application, grids, transforms, direct planar alpha composition, frame metadata, repetition, and independently decodable AV1 sequence samples exist. | Close the full color/ICC cross-product, HEVC sequence decoding, AV1/HEVC reference-dependent samples, frame-local metadata/alpha behavior, and independent animated decode vectors. | +| 5 | AV1/AVIF encoding | Not started | RGB-to-planar conversion, forward transforms, OBU writer foundations, options, and container-writing infrastructure exist. | `HeifEncoderCore` still rejects AV1. Implement a real independently decodable lossy/lossless AV1 payload and the complete AVIF item/metadata matrix. | +| 6 | HEVC/HEIC encoding | Not started | Shared input color conversion, options, and HEIF writer infrastructure exist. | `HeifEncoderCore` still rejects HEVC. Implement a real independently decodable lossy/lossless HEVC payload and the complete HEIC item/metadata matrix. | +| 7 | Animated AVIF and HEIC encoding | Not started | Image-level repetition and per-frame duration contracts exist; bounded sequence parsing defines the required container surface. | Write the bounded sequence container and encode frame dependencies, timing, repetition, alpha, presentation, color, and metadata through both completed payload encoders. | +| 8 | Hardening and release readiness | Not started | Focused tests, independent fixtures, and representative SIMD benchmarks exist for completed slices. | Close the complete verification matrix, fuzz and malformed-input coverage, all Release/API/package checks, documentation and notices, and remove every placeholder or unsupported advertised path. | + +## Immediate execution queue + +- [x] Finish the libaom-shaped AV1 forward-transform architecture, measured production dispatch, inverse-tier correction, suffix cleanup, `FeatureTestRunner` matrix, and focused Release verification recorded below. +- [ ] **Current:** complete AV1 still-image decoding for every valid still syntax path and independently verify the full bit-depth, chroma, compression-tool, filter, grain, and color matrix. +- [ ] Complete the remaining HEVC still-image profile and Range Extensions matrix with exact independent native-plane and presentation evidence. +- [ ] Close shared decoded presentation, ICC, alpha, grid, transform, metadata, and animated AV1/HEVC decode gates. +- [ ] Implement and independently verify real AV1/AVIF still encoding. +- [ ] Implement and independently verify real HEVC/HEIC still encoding. +- [ ] Implement and independently verify animated AVIF and HEIC encoding through the bounded image-sequence surface. +- [ ] Complete the public API, documentation, provenance, hardening, performance, and release gates, then run the full Release matrix. + +Performance, allocation, documentation, and independent test work are part of each delivery stage. They are not deferred cleanup phases, and the next queued stage must not begin while the current stage has unverified code or an open correctness gate. + +## Detailed implementation ledger + +- [ ] **Implemented foundation; conformance gate open:** complete HEVC still-picture reconstruction and connect its native planes to the verified shared SIMD-first HEIF color pipeline. - [x] Correct AV1 clipped-edge partition entropy handling and verify every block-mode cell from a real libavif AVIF against libaom 3.14.1. - [x] Preserve the exact ICC payload from an independently encoded AVIF primary item. - [x] Prove that a genuine non-sRGB AVIF profile changes decoded pixels and matches the independently converted source image within the documented AV1 tolerance. @@ -71,21 +110,24 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Allocate HEVC emulation-prevention positions once at the exact validated count without a growing list plus array copy. - [x] Use ImageSharp's pooled `ChunkedMemoryStream` and direct chunk writes for the legacy JPEG item payload. - [x] Keep exact owned arrays only where ICC, Exif, XMP, or retained property state must outlive pooled parser buffers. -- [x] **Completed:** eliminate per-call scratch allocation across every AV1 forward and inverse 1-D transform, replace shared mutable transform scratch with operation-owned workspace, and implement libaom-shaped bulk transform paths with scalar parity. +- [x] **Completed:** eliminate per-call scratch allocation across every AV1 forward and inverse 1-D transform, replace shared mutable transform scratch with operation-owned workspace, and complete the libaom-shaped SIMD-first transform paths with scalar fallback. - [x] Inventory the existing hot-path scratch use: 15 forward/inverse transformer types contain 30 `stackalloc int[...]` sites. - [x] Refactor every affected forward transform to reuse caller-owned workspace and its output buffer for staged ping-pong storage. - [x] Refactor every affected inverse transform to reuse caller-owned workspace and its output buffer for staged ping-pong storage. - [x] Remove the shared mutable `Av1ForwardTransformer.TemporaryCoefficientsBuffer` and give each concurrent transform operation exclusive workspace ownership. - [x] Replace the instance-transformer interface with stateless forward and inverse operator structs following the JPEG color-transform static-generic operator pattern. - [x] Select transform type, size, bit depth, and ISA once at the 2-D block boundary rather than dispatching through an interface for every row and column. - - [x] Port the applicable libaom bulk forward-transform kernels using the existing ImageSharp `Vector128` and `Vector256` infrastructure while retaining the scalar oracle. + - [x] Port the DCT4/8/16/32/64, ADST4/8/16, and identity4/8/16/32 stage networks from the pinned libaom scalar and Highway sources into one static-generic operator architecture. + - [x] Implement paired add/subtract and whole-butterfly primitives for scalar, `Vector128`, `Vector256`, and `Vector512` values, including saturated packed arithmetic and shared widening work at each supported SIMD width. + - [x] Implement the complete libaom two-dimensional load, flip, shift, axis-transform, transpose, rectangle-normalization, promotion, and 64-point coefficient-retention pipeline without per-block allocation. - [x] Port the applicable libaom bulk inverse-transform kernels using the same tables, rounding, saturation, and clipping rules as the scalar oracle. - - [x] Extend every forward and inverse DCT, ADST, identity, two-dimensional traversal, and 8/10/12-bit reconstruction operator with a managed `Vector512` path. Dispatch it first for blocks with complete sixteen-lane tiles and retain the same staged arithmetic as the scalar, `Vector128`, and `Vector256` operators. + - [x] Use normal ImageSharp descending-width dispatch and require the actual packed arithmetic ISA when selecting packed `Vector512` traversal. - [x] Document scratch ownership, stage-buffer alternation, fixed-point rounding, lane layout, transposition, and scalar fallback decisions at their implementation points. - - [x] Add complete-block BenchmarkDotNet coverage for scalar, `Vector128`, `Vector256`, and runtime-dispatched 8x8 and 32x32 forward/inverse DCT paths, plus the applicable `Vector512` 32x32 paths, including managed-allocation reporting. - - [x] Verify the scalar, `Vector128`, and `Vector256` paths across all transform types, sizes, bit depths, edge blocks, and bounded overflow cases, then record representative complete-block timings and allocations. - - Verification covers all 159 normative size/type combinations at 8, 10, and 12 bits with padded input, prediction, and destination strides. Each of the 477 configurations compares scalar and `Vector128` output, and every configuration that contains a complete eight-lane tile also compares `Vector256` output. The 989-test focused suite passes with normal AVX2 dispatch, with AVX2 disabled, and with all hardware intrinsics disabled. - - [ ] Execute the new `FeatureTestRunner` matrix for every sixteen-lane transform configuration with AVX-512 enabled and disabled, run the Release build, and record complete-block `Vector512` versus `Vector256` timings before treating the new tier as verified. + - [x] Remove the separate SIMD files, width-specific forward contracts, and sample-representation suffixes so each transform operator owns one behavior model. + - [x] Verify every 1-D operator representation and every valid 2-D size/type/bit-depth combination through `FeatureTestRunner` with normal hardware, AVX-512 disabled, AVX disabled, and all hardware intrinsics disabled. + - The focused Release run passes all 511 forward and inverse transform cases. The suite covers DCT, ADST, and identity operators, packed overflow-sensitive inputs, padded 2-D input strides, all valid transform configurations, 8/10/12-bit dispatch, inverse reconstruction, and the zero-allocation block contract. + - [x] Benchmark the production 32x32 DCT path after the complete paired stage port with preferred 256-bit and 512-bit widths. + - On the measured .NET 10 AVX-512 host, the 8-bit path measured 1.334 microseconds at 256 bits and 1.349 microseconds at 512 bits. The 12-bit path measured 2.977 microseconds at 256 bits and 2.008 microseconds at 512 bits. BenchmarkDotNet reported no managed allocation for any measured path, so production retains the normal runtime-selected width instead of a transform-type or bit-depth patch table. - [ ] **Queued:** restore bounded animated HEIC and AVIF image-sequence scope, including the required image-level and per-frame metadata contracts, without introducing unrelated ISO BMFF surfaces. - [x] Reconcile the top-level still-image-only scope with the required animated HEIC and AVIF completion boundary before sequence implementation begins. - [x] Define the ImageSharp image-level sequence metadata and per-frame metadata contracts, including observable timing, repetition, color, alpha, orientation, and profile behavior. @@ -191,7 +233,7 @@ The implementation must validate uniqueness, versions, entry counts, run totals, Use the references in this order when behavior differs: 1. The published ISO BMFF, HEIF, HEVC, AV1, AV1-ISOBMFF, and AVIF requirements are normative. -2. The official [AOM AV1 Codec Library](https://aomedia.googlesource.com/aom/) is the primary implementation reference for AV1 decode, encode, high-bit-depth behavior, tests, and optimized scalar/SIMD algorithms. Pin one reviewed commit before porting. Use its scalar C paths as behavioral references and its architecture-specific paths as SIMD references to be expressed with ImageSharp's existing managed intrinsics. +2. The official [AOM AV1 Codec Library](https://aomedia.googlesource.com/aom/) is the primary implementation reference for AV1 decode, encode, high-bit-depth behavior, tests, and optimized scalar/SIMD algorithms. Pin one reviewed commit before porting. Use its architecture-specific paths to shape the managed SIMD-first production architecture and its scalar C paths to define the equivalent fallback behavior. 3. The local `D:\GitHub\AOMediaCodec\libavif` checkout is the AVIF container, metadata, color-conversion, grid, alpha, and interoperability oracle. At inspection time it identifies itself as 1.4.2-devel. 4. The official HM reference software at commit `9c1f298659ab0cee9dc13d23d0304221575410b9` is the primary scalar implementation reference for HEVC syntax, reconstruction, and the complete Range Extensions profile matrix. Android `libhevc` v1.6.0 at commit `c83a76b084498d55f252f48b2e3786804cdf24b7` is a secondary production-C comparison for its supported common 8-bit 4:2:0 paths; its decoder rejects other chroma formats and bit depths and therefore is not the feature-coverage authority. The HEVC specification remains normative, and an independent HEIC container implementation is still required as an interoperability oracle. Do not copy from GPL or otherwise incompatible sources. 5. Existing ImageSharp codecs are the authority for ImageSharp API shape, memory ownership, stream behavior, cancellation, resource limits, pixel conversion, tests, and SIMD dispatch. @@ -219,23 +261,23 @@ This snapshot pins or classifies the available references and failures; it does | Managed implementation | Normative behavior | Reviewed implementation reference | Use | | --- | --- | --- | --- | -| `HeifColorConversionParameters`, `HeifColorConverter`, `IHeifPlanarSampleBuffer`, `HeifPlanarColorConverter`, `Av1YuvConverter`, and `HevcYuvConverter` | ITU-T H.273 (V4) equations 14-16 and 27-84, including limited/full-range scaling, chromaticity-derived equations 39-47, YCgCo equations 51-57, constant-luminance equations 66-75, IPT-C2, YCgCo-Re/YCgCo-Ro, and the PQ/HLG ICtCp matrices; AV1 section 6.4.2 and HEVC Annex E chroma sample positions | Official ITU-T H.273 (V4) (07/2024); libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for full/limited-range conversion at 8, 10, and 12 bits. Decode and encode cover every H.273 V4 matrix code point through 17: identity, YCgCo, coefficient-based NCL, both fixed and chromaticity-derived constant/non-constant-luminance systems, SMPTE ST 2085, ICtCp, IPT-C2, YCgCo-Re, and YCgCo-Ro. One closed generic planar driver owns pooled sequential row traversal, chroma reconstruction/downsampling, fixed-point specialization, SIMD dispatch, scalar tails, and packed-pixel conversion for both AV1 and HEVC adapters. Shared paths cover monochrome, YUV 4:2:0, 4:2:2, and 4:4:4; AV1 centered/co-sited placement; and all six HEVC progressive 4:2:0 sample locations. Identity and reversible lifting enforce their full-chroma constraint. Limited-range YCgCo retains the 219-code scale inherited from its R/G/B inputs instead of applying YCbCr's unrelated 224-code chroma range. YCgCo-Re and YCgCo-Ro apply range adjustment at reduced-precision RGB code values before signed integer lifting. Chromaticity derivation uses every defined H.273 primary and matches libavif's BT.709 fallback for unspecified or reserved primaries. ICtCp and IPT-C2 inverses come from the exact H.273 matrices rather than unrelated display conversions. Closed operators provide `Vector512`, `Vector256`, and `Vector128` implementations with one scalar tail. Eight-bit conversion uses JPEG's RGB plane pack/unpack contracts; high-bit-depth conversion retains native 16-bit packed pixels. | +| `HeifColorConversionParameters`, `HeifColorConverter`, `IHeifPlanarSampleBuffer`, `HeifPlanarColorConverter`, `Av1YuvConverter`, and `HevcYuvConverter` | ITU-T H.273 (V4) equations 14-16 and 27-84, including limited/full-range scaling, chromaticity-derived equations 39-47, YCgCo equations 51-57, constant-luminance equations 66-75, IPT-C2, YCgCo-Re/YCgCo-Ro, and the PQ/HLG ICtCp matrices; AV1 section 6.4.2 and HEVC Annex E chroma sample positions | Official ITU-T H.273 (V4) (07/2024); libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | SIMD-first closed operators provide full/limited-range conversion at 8, 10, and 12 bits, with an equivalent scalar fallback and tail. Decode and encode cover every H.273 V4 matrix code point through 17: identity, YCgCo, coefficient-based NCL, both fixed and chromaticity-derived constant/non-constant-luminance systems, SMPTE ST 2085, ICtCp, IPT-C2, YCgCo-Re, and YCgCo-Ro. One closed generic planar driver owns pooled sequential row traversal, chroma reconstruction/downsampling, fixed-point specialization, SIMD dispatch, scalar tails, and packed-pixel conversion for both AV1 and HEVC adapters. Shared paths cover monochrome, YUV 4:2:0, 4:2:2, and 4:4:4; AV1 centered/co-sited placement; and all six HEVC progressive 4:2:0 sample locations. Identity and reversible lifting enforce their full-chroma constraint. Limited-range YCgCo retains the 219-code scale inherited from its R/G/B inputs instead of applying YCbCr's unrelated 224-code chroma range. YCgCo-Re and YCgCo-Ro apply range adjustment at reduced-precision RGB code values before signed integer lifting. Chromaticity derivation uses every defined H.273 primary and matches libavif's BT.709 fallback for unspecified or reserved primaries. ICtCp and IPT-C2 inverses come from the exact H.273 matrices rather than unrelated display conversions. Closed operators provide `Vector512`, `Vector256`, and `Vector128` implementations with one scalar tail. Eight-bit conversion uses JPEG's RGB plane pack/unpack contracts; high-bit-depth conversion retains native 16-bit packed pixels. | | `HeifTransferFunctions` | ITU-T H.273 (V4) Table 3 transfer characteristics 1-18 | Official ITU-T H.273 (V4) (07/2024); libavif `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826` | Apply every HEVC/AV1-signallable transfer function required by constant-luminance, ICtCp, and IPT-C2 conversion. Retain the H.273 normalized PQ and HLG definitions; do not import libavif's display-oriented 203-nit scaling or HLG OOTF into codec sample interpretation. Use libavif's midpoint convention only for the non-bijective zero code of the two logarithmic curves. | | `ObuReader.ReadSequenceHeader`, `ReadUncompressedFrameHeader`, decoder-model parsing, and operating-parameter consumption | AV1 sections 5.5.2 through 5.5.4 sequence timing and decoder-model syntax, section 5.9.2 uncompressed frame-header syntax, and section 5.9.31 temporal-point syntax | libaom `av1/decoder/decodeframe.c` functions `av1_read_decoder_model_info`, `av1_read_op_parameters_info`, `read_temporal_point_info`, and `read_uncompressed_header`, plus `common/av1_config.c`, at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Read the normative 32-bit decoding-tick field; consume operating-point buffer delays and the layer-applicable frame removal times needed to keep a non-reduced still-image sequence aligned; and read presentation time only under the normative decoder-model condition. Retain no scheduling or playback behavior from those values and introduce no ISO BMFF timing, track, sample-table, or sequence surface. | | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit packed-pixel conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use ImageSharp's existing `Rgb48` pixel-operation paths in both directions so packed-pixel staging does not reduce high-bit-depth samples to eight bits. | -| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/blockd.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | +| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictor operators | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/blockd.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design the shared predictor contract, sample layout, reference preparation, and scratch ownership for widest-to-narrowest SIMD traversal, then provide the equivalent scalar fallback through that contract. Preserve luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | | `Av1TileReader` palette mode/color-map parsing, `Av1SymbolDecoder` palette distributions, `Av1BlockModeInfo` palette state, and `Av1PredictionDecoder` palette reconstruction | AV1 sections 5.11.46, 5.11.49, and 7.11.2 palette prediction | libaom `av1/decoder/decodemv.c`, `av1/decoder/detokenize.c`, `av1/decoder/decoder.h`, `av1/common/pred_common.c`, `av1/common/pred_common.h`, and `av1/common/entropymode.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Reuse the existing tile range decoder and frame-owned mode map while porting the normative palette cache merge, high-bit-depth color deltas, tile-adaptive mode/size/index distributions, diagonal color-map traversal, edge padding, and direct palette-sample reconstruction. This is AV1 still-image compression syntax and does not add retained video reference state or any ISO BMFF surface. | -| `Av1LoopFilterKernels`, `Av1LoopFilterContext`, and `Av1LoopFilterDecoder` | AV1 section 7.14 deblocking loop filter | libaom `aom_dsp/loopfilter.c` and `av1/common/av1_loopfilter.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Port the scalar 4-, 6-, 8-, and 14-tap low/high-bit-depth filters, sharpness thresholds, still-frame intra filter-level derivation, transform-edge selection, and plane traversal before enabling the stage. Later SIMD must preserve the scalar result. This is normative AV1 image reconstruction and adds neither generic ISO BMFF models nor retained video reference state. | -| `Av1CdefDecoder`, `Av1CdefKernels`, and CDEF-unit strength storage | AV1 sections 7.15.2 through 7.15.4 constrained directional enhancement filtering | libaom `av1/common/cdef.c`, `av1/common/cdef_block.c`, `av1/common/cdef.h`, and `av1/common/cdef_block.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Port the scalar direction search, variance adjustment, constrained primary/secondary taps, subsampling direction conversion, skipped-8x8 selection, and frame-edge sentinel behavior. Use a frame-owned source snapshot so filtering never consumes already modified samples. This is normative AV1 still-image reconstruction and introduces no ISO BMFF, track, timing, or sequence-playback surface. | +| `Av1LoopFilterKernels`, `Av1LoopFilterContext`, and `Av1LoopFilterDecoder` | AV1 section 7.14 deblocking loop filter | libaom `aom_dsp/loopfilter.c` and `av1/common/av1_loopfilter.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design the low/high-bit-depth edge layout and 4-, 6-, 8-, and 14-tap kernels for SIMD-first traversal, with sharpness thresholds, still-frame intra filter-level derivation, transform-edge selection, and plane traversal shared by the exact scalar fallback. Implement and verify both through the same contract before enabling the stage. This is normative AV1 image reconstruction and adds neither generic ISO BMFF models nor retained video reference state. | +| `Av1CdefDecoder`, `Av1CdefKernels`, and CDEF-unit strength storage | AV1 sections 7.15.2 through 7.15.4 constrained directional enhancement filtering | libaom `av1/common/cdef.c`, `av1/common/cdef_block.c`, `av1/common/cdef.h`, and `av1/common/cdef_block.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design direction search and constrained primary/secondary filtering around vector-friendly block storage and SIMD-first kernels, with variance adjustment, subsampling direction conversion, skipped-8x8 selection, and frame-edge sentinel behavior shared by the scalar fallback. Use a frame-owned source snapshot so filtering never consumes already modified samples. This is normative AV1 still-image reconstruction and introduces no ISO BMFF, track, timing, or sequence-playback surface. | | `Av1SuperResolutionDecoder`, `Av1SuperResolutionKernels`, frame-size derivation, and decoded-image dimensions | AV1 section 7.16 normative super-resolution upscaling | libaom `av1/common/resize.c`, `av1/common/resize.h`, `av1/common/convolve.c`, and `aom_dsp/aom_filter.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Port the fixed 64-phase, 8-tap horizontal filter, phase/step derivation, replicated frame edges, chroma width rounding, signed rounding, and 8/10/12-bit clipping. Reuse ImageSharp's existing cross-platform `Vector128_.MultiplyAddAdjacent` helper for the exact eight-coefficient dot product with a scalar fallback. Generic image resizing is not normative AV1 super-resolution. This adds no track, timing, fragment, animation, or generic ISO BMFF model. | | `Av1TileReader` loop-restoration unit syntax, `Av1SymbolDecoder` restoration distributions/subexponential codes, and `Av1FrameInfo` unit storage | AV1 section 5.11.57 `read_lr` and `read_lr_unit` syntax | libaom `av1/decoder/decodeframe.c`, `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/entropymode.c`, `aom_dsp/binary_codes_reader.c`, and `aom_dsp/recenter.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Decode tile-local switchable/Wiener/self-guided selections, finite reference-subexponential coefficients, chroma Wiener windows, self-guided parameter sets, super-resolution-adjusted unit corners, and the AV1 nearest-unit-count rule into frame-owned per-plane grids. This is compressed still-image syntax and adds no movie, track, timing, fragment, audio, or sequence surface. | -| `Av1WienerFilter` | AV1 sections 7.17.4 and 7.17.5 Wiener restoration filtering and coefficient derivation | libaom `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/convolve.c`, and `av1/common/convolve.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the implicit center-sample contribution, separable horizontal/vertical rounding, bit-depth-dependent 16-bit intermediate range, and final 8/10/12-bit clipping. Reuse `Vector128_.MultiplyAddAdjacent` for the contiguous horizontal eight-tap product with an exact scalar fallback. Keep this scalar/SIMD oracle disabled until restoration stripe boundaries and self-guided filtering are both complete. | -| `Av1SelfGuidedFilter` | AV1 sections 7.17.2 and 7.17.3 self-guided and box-filter processes | libaom `av1/common/restoration.c` and `av1/common/restoration.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the sixteen normative radius/variance parameter sets, local mean and variance normalization, alternating-row radius-two optimization, decoded projection-coefficient behavior, signed rounding, and 8/10/12-bit clipping. Use caller-owned scratch storage and a scalar oracle before considering a managed SIMD translation of libaom's architecture-specific nonlinear filter kernels. Keep this image-reconstruction stage disabled until restoration stripe boundaries are complete. | +| `Av1WienerFilter` | AV1 sections 7.17.4 and 7.17.5 Wiener restoration filtering and coefficient derivation | libaom `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/convolve.c`, and `av1/common/convolve.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the implicit center-sample contribution, separable horizontal/vertical rounding, bit-depth-dependent 16-bit intermediate range, and final 8/10/12-bit clipping. Reuse `Vector128_.MultiplyAddAdjacent` for the contiguous horizontal eight-tap product with an exact scalar fallback. Keep the restoration stage disabled until stripe boundaries and self-guided filtering are both complete. | +| `Av1SelfGuidedFilter` | AV1 sections 7.17.2 and 7.17.3 self-guided and box-filter processes | libaom `av1/common/restoration.c` and `av1/common/restoration.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Design the window layout, caller-owned scratch, local statistics, and projection traversal from libaom's architecture-specific kernels so the production path is SIMD-first. Implement the equivalent scalar fallback through the same contract. Preserve the sixteen normative radius/variance parameter sets, local mean and variance normalization, alternating-row radius-two optimization, decoded projection-coefficient behavior, signed rounding, and 8/10/12-bit clipping. Keep this image-reconstruction stage disabled until restoration stripe boundaries are complete. | | `Av1LoopRestorationBoundary`, `Av1LoopRestorationDecoder`, and `Av1FrameDecoder` restoration-stage ordering | AV1 section 7.17 loop restoration, including striped boundary semantics | libaom `av1/common/restoration.c`, `av1/common/restoration.h`, `av1/common/resize.c`, and `av1/decoder/decodeframe.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve two deblocked rows at internal 64-luma stripe boundaries before CDEF, apply the existing normative SIMD-backed super-resolution kernel to saved rows when scaled, use post-CDEF/super-resolution samples at frame edges, extend the final restoration unit up to 150 percent of nominal size, and filter from immutable plane snapshots into separate output planes. This is bounded still-image reconstruction state, not retained reference-frame, track, timing, or playback state. | | `Av1FilmGrainDecoder` and `Av1FilmGrainGaussianSequence` | AV1 section 7.18 film-grain synthesis | libaom `av1/decoder/grain_synthesis.c`, `av1/decoder/grain_synthesis.h`, and `aom_dsp/grain_params.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative 2,048-sample Gaussian sequence, linear-feedback shift register, luma/chroma autoregressive templates, scaling lookup interpolation, 32x32 block selection, boundary overlap, restricted-range clipping, monochrome and 4:2:0/4:2:2/4:4:4 layouts, and 8/10/12-bit arithmetic. Use allocator-owned scratch and runtime-optimized span copies. Apply grain only to the displayed still-image samples after all in-loop filters; reference-frame parameter inheritance remains sequence-playback state and is outside this codec scope. | | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. | | `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. | -| `Av1ForwardTransformer`, `Av1Inverse2dTransformer`, `Av1Transform2dFlipConfiguration`, the forward/inverse 1-D operator structs, and `Av1Transform1dMath` | AV1 forward transform definitions and section 7.11.2 inverse transform and reconstruction | libaom `av1/encoder/av1_fwd_txfm1d.c`, `av1/encoder/av1_fwd_txfm2d.c`, `av1/common/av1_inv_txfm1d_cfg.h`, `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, the x86 AVX2/SSE4 implementations, and the corresponding Neon implementations at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative staged DCT, ADST, and identity arithmetic, direction-specific stage ranges and shifts, transposition, clipping, and high-bit-depth sample addition. Stateless static-generic operators follow ImageSharp's JPEG color-transform pattern. `Vector128`, `Vector256`, and managed `Vector512` kernels retain the same wrapping 32-bit lane arithmetic before the bounded rounding shift, with the scalar operators as the behavioral oracle. The pinned upstream transform paths do not provide a complete AVX-512 forward, inverse, and high-bit-depth family, so the managed sixteen-lane tier must prove exact parity against the scalar path rather than claim upstream AVX-512 equivalence. | +| `Av1ForwardTransformer`, `Av1Inverse2dTransformer`, `Av1Transform2dFlipConfiguration`, the forward/inverse 1-D operator structs, and `Av1Transform1dMath` | AV1 forward transform definitions and section 7.11.2 inverse transform and reconstruction | libaom `av1/encoder/av1_fwd_txfm2d_hwy.h`, its AVX-512 and AVX2 instantiations, `av1/encoder/av1_fwd_txfm1d.c`, `av1/common/av1_inv_txfm1d_cfg.h`, `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, the x86 AVX2/SSE4 implementations, and the corresponding Neon implementations at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative staged DCT, ADST, and identity arithmetic, direction-specific stage ranges and shifts, transposition, clipping, and high-bit-depth sample addition. Stateless static-generic operators follow ImageSharp's JPEG color-transform pattern. Forward operators share one behavior model across scalar, `Vector128`, `Vector256`, and `Vector512`, selecting packed or expanded lane storage at the 2-D boundary. Inverse production traversal retains the verified scalar, `Vector128`, and `Vector256` tiers until a wider upstream shape and complete-block benefit are both established. | | `HeifDecoderCore` box extension handling and `HeifDecoderCore`/`HeifEncoderCore` item-property associations | ISO/IEC 14496-12 box extensibility and section 8.11.14 item properties and `ipma` syntax | libavif `src/read.c` and `src/write.c` at `092276ce89098ead06db80975173191e5fee1826` | Skip unrecognized top-level and metadata child boxes, preserve the position of every property in `ipco`, reject an unrecognized property only when its item association marks it essential, associate properties by item ID, and read or write the essential bit plus one-based 7-bit or 15-bit property index according to the full-box flags. Independent HEIC, HIF, and AVIF fixtures provide the reader oracle; container-level identification of encoded output guards the writer independently of pixel roundtripping. | | `HeifCleanAperture`, `HeifItem` presentation state, and `HeifDecoderCore` transformative-property parsing and application | ISO/IEC 14496-12 section 12.1.4 clean aperture; HEIF image rotation and mirror properties; MIAF section 7.3.6.7 presentation order and section 7.3.9 essential transformative properties | libavif `src/avif.c` clean-aperture conversion, `src/read.c` property parsers and alpha-property validation, and `apps/shared/avifutil.c` transform application at `092276ce89098ead06db80975173191e5fee1826` | Resolve fractional clean-aperture dimensions and center offsets to exact bounded integer pixels, validate the registered rotation/mirror reserved bits, require essential associations, crop after auxiliary-alpha composition, map counter-clockwise HEIF quarter turns to ImageSharp's optimized clockwise rotate modes, then mirror around the signaled axis. Reuse ImageSharp's existing crop, rotation, and flip processors for every pixel type. Retain only the three image-item property values; do not add a generic transform-box or ISO BMFF model. | | `HeifConstants.IsSupportedFileType`, `HeifImageFormatDetector`, and `HeifDecoderCore.CheckFileTypeBox` | ISO/IEC 14496-12 `FileTypeBox` syntax and the MP4 Registration Authority HEIF/AVIF still-image and sequence brand registrations | libavif `src/read.c` functions `avifParseFileTypeBox`, `avifFileTypeHasBrand`, and `avifFileTypeIsCompatible` at `092276ce89098ead06db80975173191e5fee1826` | Apply one rule to the major and compatible brands, accept implemented still-image brands and the bounded `avis`, `hevc`, and `hevx` image-sequence brands, and distinguish item and sequence presentation before parsing their payloads. Layered HEVC and JPEG sequence brands remain unsupported. The decoder validates the complete `ftyp` payload; the fixed-size format detector inspects the available prefix. | @@ -270,7 +312,7 @@ This table is intentionally incomplete. Add a row before each additional AV1 or ## Current implementation assessment -This assessment is based on the current source after the upstream ImageSharp merge and the baseline recorded above. Unless a result is stated explicitly, each item is a source-inspection finding rather than a verified interoperability claim. +This assessment was reconciled with the source tree on 2026-08-26. Unless a result is stated explicitly, each item is a source-inspection finding rather than a verified interoperability claim. ### Public integration @@ -284,12 +326,12 @@ This assessment is based on the current source after the upstream ImageSharp mer ### HEIF/ISO BMFF container - The bounded reader now handles basic, extended-size, UUID, and permitted top-level to-end boxes, skips unknown optional top-level and metadata children, and rejects child boxes that escape their parent. -- File type handling now evaluates supported still-image major and compatible brands while rejecting registered sequence major brands without adding track or timing support. +- File type handling evaluates supported still-image major and compatible brands and recognizes the bounded `avis`, `hevc`, and `hevx` image-sequence brands. The sequence parser retains only the selected picture and auxiliary-alpha tracks, timing, sample locations, dependencies, presentation properties, and image metadata required by animated HEIF; unrelated media tracks and general ISO BMFF playback state are not materialized. - Item IDs are resolved as keys rather than list indices; metadata children are indexed and parsed in dependency order rather than physical order. - Item property associations now preserve physical `ipco` indices, apply one-based 7-bit or 15-bit indices and essential flags, associate by item ID, and reject arbitrary unknown essential properties. - Item locations now support bounded file-relative and `idat`-relative storage, multiple ordered extents, versioned item IDs, 0/4/8-byte registered field sizes, and 64-bit offsets. Referenced-item construction method two and external data references remain explicitly unsupported. - Grid derived-image decoding now parses both registered descriptor widths, resolves the ordered `dimg` cells, validates tile count, coding format, dimensions, canvas coverage, and edge overlap, then composes the output through ImageSharp row buffers. Unsupported grid tile codecs can select only a decodable thumbnail of the same primary grid. HEVC grids have exact complete-image libheif references and exact HM tile-plane references; independent AV1 and JPEG grid fixtures are still required. -- Alpha auxiliary decoding now recognizes `auxC`, `auxl`, and `prem`, supports a direct auxiliary image/grid and the per-color-grid-tile form, preserves normalized alpha through `L16`/`Rgba64`, box-resamples differing plane sizes, and reports alpha presence. HEVC auxiliary-alpha presentation matches the independent libheif reference exactly, while independent AVIF alpha fixtures are still required against the incomplete AV1 reconstruction pipeline. +- Alpha auxiliary decoding recognizes `auxC`, `auxl`, and `prem`, supports direct auxiliary images, auxiliary grids, and per-color-grid-tile alpha, and reports alpha presence. The common-size path normalizes the native luma plane and composes each row directly into the destination; the scaled path uses allocator-owned row staging and the bounded ImageSharp resize window instead of materializing a complete `Image`. HEVC auxiliary-alpha presentation matches the independent libheif reference exactly, and independently encoded AVIF alpha fixtures cover composed alpha and ICC interaction. - Clean aperture, image rotation, and image mirror properties now validate their registered payloads, exact integer crop geometry, and essential associations; affect Identify dimensions; and reuse ImageSharp's optimized crop/rotate/flip processors after auxiliary alpha composition in the MIAF-defined order. Independent transform vectors must still verify every crop/rotation/mirror/alpha combination. - Decode now resolves `cdsc`-associated Exif and `application/rdf+xml` XMP items for the primary still image, validates the declared Exif TIFF-header offset, and attaches the payloads through ImageSharp's existing profile types before presentation transforms. Independent AVIF, HEIC, and HIF metadata fixtures and Identify-time profile reporting remain required. - ICC and `nclx` CICP color properties are now associated with the presented color item instead of global parser state, validated, and exposed on Decode and Identify through the existing ImageSharp profiles. AV1 decode requires specified sequence-header fields and the range flag to match `nclx`, uses the property only for fields the bitstream leaves unspecified, and exposes the effective sequence-header CICP description when no property is present. Independent libavif fixtures verify exact ICC byte preservation, observable non-sRGB conversion, canonical-sRGB compaction, non-sRGB compact preservation, and metadata skipping for primary, grid, alpha-composited, and every presented sequence frame; the alpha vector also verifies that ICC conversion does not alter composed alpha. Identify-time bitstream fallback, independent CICP vectors, and HEVC integration remain required. @@ -301,8 +343,8 @@ This assessment is based on the current source after the upstream ImageSharp mer ### HEVC decoder and encoder - `Heif4CharCode` recognizes `hvc1` image items, `HevcCodecConfiguration` validates and associates the bounded `hvcC` property, Identify reports its HEVC precision and monochrome shape, and `HeifCompressionFactory` registers the bounded HEVC still-image decoder. -- The HEVC path validates NAL headers and RBSP escaping, links `hvcC` VPS/SPS/PPS structures through Range Extensions, parses one length-delimited base-layer IDR picture, and reconstructs its complete intra-coded coding tree. It includes CABAC, transform traversal, coefficient decoding, intra prediction, inverse quantization and transform, PCM, deblocking, sample-adaptive offset, and allocator-owned native `ushort` planes for monochrome, 4:2:0, 4:2:2, 4:4:4, separate-color-plane, and 8-through-16-bit samples. The item adapter converts the SPS conformance window directly through the shared pooled SIMD-first H.273 pipeline and preserves the effective container-or-VUI CICP description. Four real item payloads match HM 18 native planes exactly, covering 8-bit 4:2:0 grids, cropping, 8-bit 4:4:4, and wavefront entry points. Complete images covering grids, auxiliary alpha, ICC, 4:2:0, and 4:4:4 presentation also match independent libheif output exactly. High bit depths, monochrome, 4:2:2, remaining Range Extensions tools, and the complete color matrix still require independent fixtures before HEVC conformance is established. -- There is no HEVC encoder. The current HEIC-branded encoder writes a legacy JPEG payload and therefore cannot provide HEIC output. +- The HEVC path validates NAL headers and RBSP escaping, links `hvcC` VPS/SPS/PPS structures through Range Extensions, parses one length-delimited base-layer IDR picture, and reconstructs its complete intra-coded coding tree. It includes CABAC, transform traversal, coefficient decoding, intra prediction, inverse quantization and transform, PCM, deblocking, sample-adaptive offset, and allocator-owned native `ushort` planes for monochrome, 4:2:0, 4:2:2, 4:4:4, separate-color-plane, and 8-through-16-bit samples. The item adapter converts the SPS conformance window directly through the shared pooled SIMD-first H.273 pipeline and preserves the effective container-or-VUI CICP description. Four real item payloads match HM 18 native planes exactly, and the first pictures from 10 official Sony GENERAL Range Extensions streams match every 8/10/12-bit monochrome, 4:2:0, 4:2:2, and 4:4:4 sample with normal and forced-scalar dispatch. Complete images covering grids, auxiliary alpha, ICC, 4:2:0, and 4:4:4 presentation also match independent libheif output exactly. Profiles and Range Extensions tools not isolated by that matrix, plus the complete color matrix, remain open. +- There is no HEVC encoder. `HeifEncoderCore` rejects `HeifCompressionMethod.Hevc`; the separately selected legacy JPEG path emits a generic HEIF/HIF-style JPEG image item and does not provide HEIC output. - Permanent HEVC conformance tests compare extracted item planes exactly with HM output and complete presented images exactly with pinned libheif/libde265 reference PNGs. The fixtures do not yet cover the complete exposed profile matrix. ### AV1 decoder @@ -310,37 +352,36 @@ This assessment is based on the current source after the upstream ImageSharp mer - The single-still `Av1Decoder` path now parses tile state before allocating and reconstructing one independently decodable frame, and it disposes the reconstruction planes after pixel conversion. It deliberately does not retain animation/video reference frames or implement `show_existing_frame` playback state. - Transform coefficient entropy derivation and updates now address the above contexts relative to the tile column and the left contexts relative to the current superblock row, preserve luma coordinates independently of chroma subsampling, and test every packed context entry for the libaom any-nonzero rule. Extended vertical partition updates advance the mode-information column rather than the row. The existing multi-superblock 4:4:4 AVIF fixture now completes tile parsing; independent coefficient-context vectors across tile boundaries, chroma layouts, bit depths, and edge-clipped transforms remain required. - The reconstruction pipeline now records plane-relative transform geometry, preserves tile-local delta-Q and delta-LF predictors, derives segmentation and reference-adjusted filter levels, and runs the exact scalar low/high-bit-depth AV1 4-, 6-, 8-, and 14-tap deblocking kernels in normative vertical-then-horizontal order. It then applies scalar CDEF direction search, luma variance adjustment, primary and secondary constrained taps, chroma direction conversion, high-bit-depth scaling, skipped-block selection, and frame-edge sentinel handling from immutable per-plane snapshots. Active super-resolution derives the Appendix A bounded coded width and applies the exact 64-phase, 8-tap horizontal filter with aligned reconstruction-edge input, 8/10/12-bit clipping, and the existing cross-platform `Vector128_.MultiplyAddAdjacent` helper. Loop restoration follows super-resolution, preserves the required pre-CDEF deblocked context at internal stripes, and applies decoded Wiener or self-guided units from immutable plane snapshots. The visible still-image path then applies the complete self-contained film-grain parameter set after all in-loop filters. Independent 8-, 10-, and 12-bit AVIF vectors exercising every active filter and grain stage remain required before these paths have external pixel-level verification. -- Palette mode now reads the normative luma/chroma mode and size CDFs, neighbor color caches, high-bit-depth color syntax, diagonal color-index maps, clipped-edge padding, and direct sample prediction through the existing reconstruction pipeline. The scalar implementation matches the pinned libaom source, but an independently encoded palette AVIF fixture is still required before this path is independently verified. +- Palette mode now reads the normative luma/chroma mode and size CDFs, neighbor color caches, high-bit-depth color syntax, diagonal color-index maps, clipped-edge padding, and direct sample prediction through the existing reconstruction pipeline. The current scalar behavior matches the pinned libaom source, but this path is not production-complete until it is expressed through the SIMD-first predictor architecture and verified with an independently encoded palette AVIF fixture. - Non-reduced still-image sequence parsing now consumes decoder-model operating parameters, temporal presentation fields, and OBU-layer-applicable buffer-removal fields only to preserve AV1 bit alignment. The scheduling values are not retained, and no movie, track, timing, playback, or generic ISO BMFF surface has been introduced. Existing focused sequence-header coverage exercises only reduced-still syntax, so an independent non-reduced still AVIF vector remains required. - Loop-restoration unit parsing records tile-local switchable/Wiener/self-guided filter selections and coefficients in frame-owned plane grids, including super-resolution-adjusted unit corners and the corrected conditional 64x64-superblock unit-size bit. The active restoration stage implements the normative unit geometry, striped deblocked boundaries, Wiener filtering, self-guided projection, and 8/10/12-bit clipping, while reusing the existing SIMD-backed super-resolution and adjacent multiply/add primitives. Independently encoded fixtures covering every parameter set, plane layout, bit depth, and frame-edge geometry are still required. Other normative independently decodable still-image syntax paths still contain `NotImplementedException` or equivalent unsupported branches. Tile-local palette CDF adaptation is present; the remaining still-image frame-context behavior requires a separate source audit without introducing sequence playback state. - The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active intra-prediction, inverse-transform, and block-reconstruction path selects native 16-bit samples for 10/12-bit frames and has focused pipeline wiring coverage. Chroma-from-luma storage, subsampling, parameter derivation, U/V sharing, and 8/10/12-bit prediction are active; independently encoded high-bit-depth and chroma-from-luma AVIF conformance files are still required. - `Av1YuvConverter` now consumes the signaled full or limited range, every non-reserved AV1 H.273 matrix coefficient, transfer characteristics where the matrix definition requires them, subsampling, and chroma sample position for 8, 10, and 12-bit output. Its high-bit-depth decode and encode paths use allocator-backed `Rgb48` rows and the existing `PixelOperations` conversions, avoiding the former eight-bit intermediate. Encoder conversion covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with libavif-compatible box averaging. Identity, full/limited-range YCgCo, the fixed non-constant-luminance matrices, both fixed and chromaticity-derived constant/non-constant-luminance systems, SMPTE ST 2085, and PQ/HLG ICtCp are active in both directions. Independent vectors for every matrix, transfer, range, bit depth, sampling layout, and chroma position remain required before the complete color matrix is externally verified. -- Forward and inverse transforms use operation-owned allocator workspace and stateless static-generic operator structs for every valid DCT, ADST, and identity size. Named configuration factories keep the encoder's three shifts and variable cosine precision separate from the decoder's two shifts, fixed 12-bit cosine precision, and 8/10/12-bit clamp ranges. The two-dimensional traversal now selects `Vector512`, then `Vector256`, then `Vector128`, with scalar only when hardware vectorization is unavailable. The managed sixteen-lane tier covers every one-dimensional operator, both two-dimensional traversals, byte reconstruction, and high-bit-depth reconstruction without per-block allocation. Its `FeatureTestRunner` parity matrix and permanent 32x32 benchmark cases are implemented but have not yet been executed. The existing verified baseline still covers all 159 normative size/type combinations at every supported bit depth with scalar, `Vector128`, and applicable `Vector256` paths; the 989-test suite passes with normal AVX2 dispatch, AVX2 disabled, and all hardware intrinsics disabled. On the measured .NET 10 AVX2 host, runtime-dispatched 8x8 DCT blocks take 102.42 ns forward and 112.14 ns inverse versus 441.19 ns and 602.52 ns scalar; 32x32 blocks take 1.439 microseconds forward and 8.825 microseconds inverse versus 9.112 microseconds and 23.455 microseconds scalar. BenchmarkDotNet reports no managed allocation for those measured paths. -- The production prediction and nonlinear self-guided paths remain predominantly scalar. Transform traversal is SIMD-first on supported hardware, while normative super-resolution and Wiener horizontal products reuse ImageSharp's cross-platform adjacent multiply/add SIMD helper with exact scalar fallbacks; further SIMD work must preserve these scalar reconstruction oracles. +- Forward and inverse transforms use operation-owned workspace, explicit sequential fixed storage, and stateless static-generic operator structs for every valid DCT, ADST, and identity size. Named configuration factories keep the encoder's three shifts and variable cosine precision separate from the decoder's two shifts, fixed 12-bit cosine precision, and 8/10/12-bit clamp ranges. Forward traversal shares one libaom-shaped stage network across scalar, `Vector128`, `Vector256`, and `Vector512`; inverse production traversal uses the verified scalar, `Vector128`, and `Vector256` tiers. The focused 511-case Release matrix passes across the `FeatureTestRunner` hardware configurations, and the production 32x32 benchmark reports zero allocation with the paired stage implementation. +- The production prediction and nonlinear self-guided paths remain predominantly scalar and therefore do not meet the completion architecture. They must be redesigned around vector-friendly storage, shared operator contracts, and SIMD-first traversal with benchmark-gated lane selection, with their current results retained only as fallback/parity behavior. Transform traversal is already SIMD-first, while normative super-resolution and Wiener horizontal products reuse ImageSharp's cross-platform adjacent multiply/add SIMD helper with exact scalar fallbacks. ### AV1 encoder -- `HeifEncoderCore.Encode()` now remains synchronous and waits for its temporary JPEG item encoding, so work and exceptions cannot outlive the ImageSharp encoder contract. -- The current container encoder compresses pixels with the ImageSharp JPEG encoder and writes that payload into a HEIC-branded HEIF file. It does not produce AVIF. +- `HeifEncoderCore.Encode()` is synchronous, matching the ImageSharp encoder contract. +- The production switch writes only an explicitly selected legacy JPEG image item. It throws for `HeifCompressionMethod.Av1` and `HeifCompressionMethod.Hevc`, so it produces neither AVIF nor HEIC. - `Av1FrameEncoder.Encode()` is an outline of an SVT-style pipeline rather than an implementation. - Required mode-decision, block-geometry, forward-transform, token-writing, neighbor-context, palette, intra-block-copy, transform-size, quantization/rate-control, and OBU-writing paths are absent or throw. -- The current encoder test is a self-round-trip through the JPEG-in-HEIF path. It does not prove that the output is AVIF or that an independent decoder can read it. +- Encoder tests currently cover the public option contract and legacy JPEG boundaries. There is no AV1 or HEVC payload output to cross-decode. ### Legacy JPEG image items - A JPEG item decoder exists, and the Fujifilm `.hif` fixture is identified and decoded as a legacy JPEG image item. -- The encoder currently uses JPEG as an accidental fallback for all output rather than as an explicitly selected, correctly branded HEIF image-item codec. -- The legacy JPEG path needs container interoperability tests and explicit public option semantics, but the existing ImageSharp JPEG codec should remain the payload implementation. +- The encoder uses the existing ImageSharp JPEG codec only when `HeifCompressionMethod.LegacyJpeg` is explicitly selected and emits a `jpeg` primary item. +- The legacy JPEG path still needs independent container interoperability coverage, but its quality, bit-depth, lossless, and chroma restrictions are explicit at the public option and codec boundaries. ### Tests -- The repository contains HEIC, HIF, and AVIF assets, but only the legacy JPEG HIF path reaches a full reference-image comparison. Pinned upstream libavif animations now verify real 8-bit, 12-bit, alpha, Exif/XMP, repetition, and unrelated-audio sequence parsing without claiming inter-frame pixel reconstruction. -- HEVC fixtures are identified but not decoded, and there are no HEVC algorithm tests. -- The strongest AV1 integration test now drives the single-still decoder through tile parsing, reconstruction, and pixel conversion, but only verifies non-zero output rather than independent reference pixels. -- Several full-image, inverse-transform, entropy, and frame-header cases are disabled or commented out. -- Existing bitstream, predictor, transform, and entropy unit tests are useful foundations, but many compare two in-tree implementations with the same assumptions. -- There is no decode matrix covering bit depth, subsampling, range, matrix coefficients, alpha, grids, transformations, metadata, truncated data, or resource limits. -- There is no cross-codec encode test in which libavif decodes ImageSharp output, or ImageSharp decodes independently encoded output. +- HEVC coverage includes exact native-plane comparison with HM output, exact complete-image comparison with pinned libheif/libde265 references, and the 10 official Sony GENERAL Range Extensions first-picture fixtures across 8/10/12-bit monochrome, 4:2:0, 4:2:2, and 4:4:4 reconstruction. The remaining exposed profiles and individual Range Extensions tools still need exact independent vectors. +- AV1 has focused bitstream, prediction, entropy, reconstruction, filter, film-grain, color, and transform coverage, plus real libavif inputs. Valid still-image syntax paths still contain explicit unsupported branches, so the independent AV1 decode matrix is not complete. +- The AV1 transform matrix verifies scalar, `Vector128`, `Vector256`, and `Vector512` forward representations plus the production inverse tiers across every valid size/type combination and supported bit depth. All 511 focused forward and inverse cases pass in Release; `FeatureTestRunner` isolates hardware tiers, every two-dimensional configuration exercises production dispatch, and the complete-block benchmark records zero managed allocation. +- Independent libavif fixtures cover primary, grid, auxiliary-alpha, ICC, metadata-skipping, and all-sync image-sequence presentation. Reference-dependent AV1 and HEVC sequence reconstruction and independent HEVC ICC sequence coverage remain open. +- Focused decoder-option tests cover strict, ancillary-only, image-data, and metadata-skipping behavior for still items and sequence samples. Complete adversarial dimension, allocation, malformed-container, and resource-limit coverage remains open. +- No cross-codec encode test exists for AV1 or HEVC because both production encoding branches still throw before producing a payload. ## Architectural direction @@ -368,7 +409,7 @@ Use existing ImageSharp buffered stream and allocation abstractions. Do not copy Keep HEVC and AV1 bitstream state in separate codec implementations. Within each codec, separate parameter/sequence state, frame or picture headers, tile/slice entropy state, and reconstructed images. Still-image operations retain no reference pictures; bounded image-sequence operations retain only the reference frames required by their declared dependencies. Give each allocation one owner and a deterministic disposal point. -Represent 8-bit samples with bytes and high-bit-depth samples with unsigned 16-bit storage. Plane dimensions and strides must reflect monochrome and chroma subsampling instead of pretending every plane is full-resolution 4:4:4. Keep scalar reconstruction as the behavioral oracle for every SIMD implementation. +Represent 8-bit samples with bytes and high-bit-depth samples with unsigned 16-bit storage. Plane dimensions and strides must reflect monochrome and chroma subsampling instead of pretending every plane is full-resolution 4:4:4. Design storage and reconstruction operators for SIMD-first traversal, with the scalar fallback sharing the same layout and contract for behavioral parity. ### Decode directly into ImageSharp pixels @@ -446,7 +487,7 @@ Exit gate: - [ ] The parser resolves each current HEIC, HIF, and AVIF asset into a stable logical model, malformed inputs fail without escaping bounds or allocating attacker-controlled sizes, and Identify has reference-verified metadata and payload classification. -### Phase 3: complete scalar AV1 still-image reconstruction +### Phase 3: complete SIMD-first AV1 still-image reconstruction Implement and verify in dependency order: @@ -461,7 +502,7 @@ Implement and verify in dependency order: - [ ] Loop restoration. - [ ] Frame padding and film-grain synthesis when signaled. -For each item, first add a small scalar unit test against normative or independent vectors, then enable it in the frame pipeline. Remove constant feature-disable flags and unsupported branches only when their replacement is verified. Unsupported syntax must produce a codec-specific invalid-image error; it must never silently skip a normative reconstruction stage. +For each SIMD-suitable item, design the data layout, operator contract, scratch ownership, traversal, and descending-width dispatch around the production SIMD path first. Implement the behaviorally equivalent scalar fallback through the same contract, then verify every supported vector width and the scalar path against normative or independent results before enabling the stage. Serial adaptive work such as entropy decoding remains scalar where data dependencies prevent meaningful vectorization. Remove constant feature-disable flags and unsupported branches only when their replacement is verified. Unsupported syntax must produce a codec-specific invalid-image error; it must never silently skip a normative reconstruction stage. Exit gate: @@ -580,7 +621,7 @@ Exit gate: ### Phase 8: SIMD and allocation optimization -Production hot paths are designed SIMD-first. Each SIMD implementation is verified against an independent specification-shaped scalar oracle before it is connected to the decoder pipeline. +Production hot paths are designed and implemented SIMD-first. The same architecture must provide a behaviorally equivalent scalar fallback for unsupported hardware, remainder work, and parity verification; scalar code is not a temporary production architecture that SIMD is added to later. Every supported hardware path and the scalar fallback are verified against independent specification-shaped results before the subsystem is complete. Tasks: @@ -596,8 +637,8 @@ Tasks: - [ ] Implement vector paths only for confirmed hot loops, using existing `Vector128`, `Vector256`, and `Vector512` helper and dispatch patterns where supported. - [ ] Prioritize shared color conversion and pixel packing, chroma upsampling, inverse-transform add-and-clip, intra predictors, HEVC deblock/SAO, AV1 loop filter/CDEF/restoration, and contiguous grid copies. - [ ] Benchmark the complete decode color pipeline on representative 8/10/12-bit AVIF and HEIC images with and without embedded ICC profiles. Report absolute end-to-end timings and allocations in addition to the isolated YUV/CICP and ICC stage costs. -- [ ] Port upstream SIMD algorithms only after mapping lane width, signedness, intermediate precision, rounding, saturation, edge extension, and high-bit-depth behavior to the scalar oracle. -- [ ] Keep one scalar implementation as the specification-shaped reference. Vector paths must share tables and constants with it rather than duplicate codec policy. +- [ ] Design each hot-path operator and data layout from the applicable upstream SIMD algorithm, mapping lane width, signedness, intermediate precision, rounding, saturation, edge extension, and high-bit-depth behavior before implementation. +- [ ] Implement the scalar fallback through the same operator contract, tables, constants, scratch ownership, and traversal as the SIMD paths so it remains a specification-shaped oracle without becoming a separate architecture. - [ ] Test scalar and each available hardware path with intrinsics explicitly enabled and disabled, including widths shorter than a vector, exact-vector widths, non-multiples, edges, maximum sample values, and high-bit-depth overflow cases. - [ ] Remove dead or commented SIMD experiments once a verified production path replaces them. @@ -650,6 +691,7 @@ No valid HEVC or AV1 color, compression, or bit-depth row may remain `unsupporte ## Working rules for implementation - Keep changes vertical and reviewable. A slice should add one behavior, its focused tests, independent evidence, and any required notice update. +- Design SIMD-suitable codec work SIMD-first. Establish vector-friendly storage, operator boundaries, scratch ownership, traversal, every applicable lane width, and benchmark-gated dispatch before implementing the equivalent scalar fallback; never build a scalar production architecture and bolt SIMD onto it later. - Inspect every owning method and upstream invariant before adding guards. Validate external file data at the parser/model boundary and rely on those established invariants internally. - Do not extract one-use helpers merely to label code. Extract shared primitives only when they have genuine reuse or remove substantial complexity. - Put comments at the points where HEVC/AV1 rounding, edge extension, context propagation, or SIMD lane behavior is not evident from the code. Comments should explain why the algorithm has that shape and identify the normative rule and pinned implementation reference. @@ -659,18 +701,16 @@ No valid HEVC or AV1 color, compression, or bit-depth row may remain `unsupporte ## Recommended implementation order -The critical path is: - -- [ ] Baseline and provenance. -- [ ] Public HEIC/AVIF boundaries and the shared parsed HEIF container model. -- [ ] SIMD-first AV1 still decode with specification-shaped scalar oracles, starting with 8-bit 4:2:0 and completing every AVIF profile/tool. -- [ ] SIMD-first HEVC still decode with specification-shaped scalar oracles, starting with 8-bit 4:2:0 and completing every exposed HEIC profile/tool. -- [ ] Shared color, alpha, grids, metadata, and presentation transforms across all bit depths and chroma formats. -- [ ] Real AV1 encoder and libavif/libaom cross-decode across the complete matrix. -- [ ] Real HEVC encoder and independent HEIC cross-decode across the complete matrix. -- [ ] Measured SIMD/allocation work integrated with each production hot-path subsystem before that subsystem is considered complete. -- [ ] Hardening and release gates. +The dashboard and immediate execution queue define the remaining critical path. In phase terms, work proceeds as follows: -This sequence does not define partial PR completion. The early 8-bit 4:2:0 decoders and smallest-valid encoders are temporary vertical slices, but the PR remains incomplete until all nine steps and the complete still-image verification matrix pass. +- [x] Finish the Phase 8 checkpoint for the implemented AV1 `Vector512` transforms, including Release, feature-isolation, and benchmark evidence. +- [ ] **Current:** close Phase 3 by removing every unsupported valid AV1 still-image syntax path and proving the complete AVIF decode matrix with independent inputs and scalar/SIMD parity. +- [ ] Close Phase 4 by completing the remaining HEVC profile and Range Extensions matrix with exact native-plane and presented-image evidence. +- [ ] Close Phase 5 and the decode portion of the bounded sequence ledger: color, ICC, alpha, grids, presentation transforms, reference-dependent samples, and complete animated AVIF/HEIC decode. +- [ ] Close the still-image portions of Phases 0, 1, and 2 that remain as release gates: documentation, provenance, public format boundaries, API review, parser hardening, and malformed-input coverage. +- [ ] Complete Phase 6 with a real AV1 encoder and independent AVIF cross-decode across every public option and metadata combination. +- [ ] Complete Phase 7 with a real HEVC encoder and independent HEIC cross-decode across every public option and metadata combination. +- [ ] Complete animated AVIF and HEIC writing only after both still payload encoders pass their independent exit gates. +- [ ] Close Phase 8 end-to-end performance and allocation budgets for every completed subsystem, then close Phase 9 and the full verification matrix. -Do not begin by optimizing the current end-to-end pipeline: it cannot yet produce a correct HEIC or AVIF image, and several current data structures encode 8-bit 4:4:4 assumptions. Establish the correct scalar storage and behavior first so the reused or ported AOM/HEVC SIMD algorithms have a trustworthy managed oracle. +The phase numbers describe technical workstreams; they are not permission to work out of order. SIMD, allocation control, documentation, and independent evidence are completion requirements inside each active workstream. The PR remains incomplete until every dashboard stage and verification-matrix row is complete. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs index 611fa24b0..2a51b57c2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs @@ -46,22 +46,4 @@ internal readonly struct Av1ByteInverseTransformOutputOperator : IAv1InverseTran Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt64().ToScalar()); _ = bitDepth; } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref byte prediction, ref byte destination, Vector512 residual, int bitDepth) - { - // Residuals occupy signed 32-bit lanes, so widen the sixteen packed predictions before adding. The clamp - // then guarantees that both narrowing steps preserve the reconstructed byte values exactly. - Vector128 packed = Vector128.LoadUnsafe(ref prediction); - (Vector128 predicted16Lower, Vector128 predicted16Upper) = Vector128.Widen(packed); - Vector256 predicted32Lower = Vector256.Create(Vector128.WidenLower(predicted16Lower), Vector128.WidenUpper(predicted16Lower)).AsInt32(); - Vector256 predicted32Upper = Vector256.Create(Vector128.WidenLower(predicted16Upper), Vector128.WidenUpper(predicted16Upper)).AsInt32(); - Vector512 predicted32 = Vector512.Create(predicted32Lower, predicted32Upper); - Vector512 reconstructed = Vector512.Clamp(predicted32 + residual, Vector512.Zero, Vector512.Create((int)byte.MaxValue)); - Vector256 reconstructed16 = Vector256.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16.GetLower(), reconstructed16.GetUpper()); - reconstructed8.StoreUnsafe(ref destination); - _ = bitDepth; - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs index fcdca7c90..de471deba 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -34,7 +35,8 @@ internal static class Av1ForwardTransformer { Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); Guard.MustBeSizedAtLeast(workspace, Av1TransformWorkspace.GetRequiredLength(transformSize), nameof(workspace)); - DispatchColumn(input, coefficients, stride, ref config, workspace); + + DispatchColumn(input, coefficients, stride, bitDepth, ref config, workspace); } /// @@ -44,46 +46,47 @@ internal static class Av1ForwardTransformer Span input, Span coefficients, uint stride, + int bitDepth, ref Av1Transform2dFlipConfiguration config, Span workspace) { switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - DispatchRow(input, coefficients, stride, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeColumn} column transform is not valid for {config.TransformSize}."); @@ -97,47 +100,48 @@ internal static class Av1ForwardTransformer Span input, Span coefficients, uint stride, + int bitDepth, ref Av1Transform2dFlipConfiguration config, Span workspace) - where TColumnOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1ForwardTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - Transform2d(input, coefficients, stride, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeRow} row transform is not valid for {config.TransformSize}."); @@ -145,491 +149,663 @@ internal static class Av1ForwardTransformer } /// - /// Applies the specialized operator pair using the widest lane width supported by the block and processor. + /// Applies the specialized operator pair using the sample representation selected for the coded bit depth. /// private static void Transform2d( Span input, Span coefficients, uint stride, + int bitDepth, ref Av1Transform2dFlipConfiguration config, Span workspace) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TRowOperator : struct, IAv1ForwardTransform1dOperator { - int width = config.TransformSize.GetWidth(); - int height = config.TransformSize.GetHeight(); - - if (Vector512.IsHardwareAccelerated && width >= Vector512.Count && height >= Vector512.Count) + // Highway keeps eight-bit transform stages in Int16 lanes and promotes only the large rectangular layouts. + // Scalar libaom uses Int32, so hardware without packed Int16 support follows that exact fallback instead. + if (bitDepth == 8 && Vector128.IsHardwareAccelerated) { - Transform2dVector512(input, coefficients, stride, ref config, workspace); + TransformPacked(input, coefficients, stride, ref config, workspace); return; } - if (Vector256.IsHardwareAccelerated && width >= Vector256.Count && height >= Vector256.Count) - { - Transform2dVector256(input, coefficients, stride, ref config, workspace); - return; - } + TransformExpanded(input, coefficients, stride, ref config, workspace); + } + + /// + /// Applies the libaom Int16 stage pipeline used for eight-bit residuals. + /// + private static void TransformPacked( + Span input, + Span coefficients, + uint stride, + ref Av1Transform2dFlipConfiguration config, + Span workspace) + where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TRowOperator : struct, IAv1ForwardTransform1dOperator + { + int width = config.TransformSize.GetWidth(); + int height = config.TransformSize.GetHeight(); + int blockLaneCount = Avx512BW.IsSupported ? Vector512.Count : Avx2.IsSupported ? Vector256.Count : Vector128.Count; + int blockWidth = Math.Max(width, blockLaneCount); + int blockHeight = Math.Max(height, blockLaneCount); + int blockArea = blockWidth * blockHeight; + int packedBlockLength = (blockArea + 1) / 2; + bool promote = (blockWidth == 64 && blockHeight >= 32) || (blockWidth >= 32 && blockHeight == 64); + int dataOffset = Av1TransformWorkspace.Vector512StorageLength; + Span buffer0 = MemoryMarshal.Cast(workspace.Slice(dataOffset, packedBlockLength)); + ref short buffer0Base = ref MemoryMarshal.GetReference(buffer0); + + LoadPacked(input, stride, ref buffer0Base, blockWidth, width, height, config.Shift0, config.FlipUpsideDown, config.FlipLeftToRight); + TransformPackedAxis(buffer0, height, width, blockWidth, config.CosBitColumn, workspace); + + int retainedHeight = Math.Min(height, 32); + int retainedWidth = Math.Min(width, 32); + bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; - if (Vector128.IsHardwareAccelerated) + if (promote) { - Transform2dVector128(input, coefficients, stride, ref config, workspace); + Span buffer1 = workspace.Slice(dataOffset + packedBlockLength, blockArea); + int scratchOffset = dataOffset + packedBlockLength + blockArea; + Span scratch = workspace.Slice(scratchOffset); + ref int buffer1Base = ref MemoryMarshal.GetReference(buffer1); + + TransposeAndPromote( + ref buffer0Base, + blockWidth, + ref buffer1Base, + blockHeight, + width, + height, + -config.Shift1, + scratch); + + TransformExpandedAxis(buffer1, width, retainedHeight, blockHeight, config.CosBitRow, workspace); + ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); + + TransposeExpanded( + ref buffer1Base, + blockHeight, + ref coefficientBase, + retainedWidth, + retainedHeight, + retainedWidth, + -config.Shift2, + normalizeRectangle, + scratch); + return; } - Transform2dScalar(input, coefficients, stride, ref config, workspace); + Span buffer1Packed = MemoryMarshal.Cast(workspace.Slice(dataOffset + packedBlockLength, packedBlockLength)); + int packedScratchOffset = dataOffset + (2 * packedBlockLength); + Span packedScratch = workspace.Slice(packedScratchOffset); + ref short buffer1PackedBase = ref MemoryMarshal.GetReference(buffer1Packed); + + TransposePacked( + ref buffer0Base, + blockWidth, + ref buffer1PackedBase, + blockHeight, + width, + height, + -config.Shift1, + false, + packedScratch); + + TransformPackedAxis(buffer1Packed, width, height, blockHeight, config.CosBitRow, workspace); + + // The second transform produces horizontal frequency in rows and vertical frequency in lanes. Transposing + // once more adapts libaom's native layout to the row-major coefficient contract used by ImageSharp. + TransposePacked( + ref buffer1PackedBase, + blockHeight, + ref buffer0Base, + retainedWidth, + retainedHeight, + retainedWidth, + -config.Shift2, + normalizeRectangle, + packedScratch); + + StorePacked(ref buffer0Base, retainedWidth, retainedHeight, coefficients); } /// - /// Applies both transform axes with sixteen samples packed into each SIMD vector. + /// Applies the libaom Int32 stage pipeline used for high-bit-depth residuals and scalar fallback. /// - /// The one-dimensional operator applied down each column. - /// The one-dimensional operator applied across each row. - /// The spatial residual samples. - /// The destination transform coefficients. - /// The number of input samples between rows. - /// The transform dimensions, operators, flips, and fixed-point settings. - /// The reusable storage for SIMD vectors and transposed coefficients. - public static void Transform2dVector512( + private static void TransformExpanded( Span input, - Span output, - uint inputStride, + Span coefficients, + uint stride, ref Av1Transform2dFlipConfiguration config, Span workspace) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TRowOperator : struct, IAv1ForwardTransform1dOperator { - const int laneCount = 16; - const int vectorLength = Av1Constants.MaxTransformSize * laneCount; - int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); - int shift0 = config.Shift0; - int shift1 = config.Shift1; - int shift2 = config.Shift2; + int blockLaneCount = Vector512.IsHardwareAccelerated ? Vector512.Count : Vector256.IsHardwareAccelerated ? Vector256.Count : Vector128.IsHardwareAccelerated ? Vector128.Count : 1; + int blockWidth = Math.Max(width, blockLaneCount); + int blockHeight = Math.Max(height, blockLaneCount); + int blockArea = blockWidth * blockHeight; + int dataOffset = Av1TransformWorkspace.Vector512StorageLength; + Span buffer0 = workspace.Slice(dataOffset, blockArea); + Span buffer1 = workspace.Slice(dataOffset + blockArea, blockArea); + Span scratch = workspace.Slice(dataOffset + (2 * blockArea)); + ref int buffer0Base = ref MemoryMarshal.GetReference(buffer0); + ref int buffer1Base = ref MemoryMarshal.GetReference(buffer1); + + LoadExpanded(input, stride, ref buffer0Base, blockWidth, width, height, config.Shift0, config.FlipUpsideDown, config.FlipLeftToRight); + TransformExpandedAxis(buffer0, height, width, blockWidth, config.CosBitColumn, workspace); + + TransposeExpanded( + ref buffer0Base, + blockWidth, + ref buffer1Base, + blockHeight, + width, + height, + -config.Shift1, + false, + scratch); + + int retainedHeight = Math.Min(height, 32); + int retainedWidth = Math.Min(width, 32); + TransformExpandedAxis(buffer1, width, retainedHeight, blockHeight, config.CosBitRow, workspace); bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; + ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); + + TransposeExpanded( + ref buffer1Base, + blockHeight, + ref coefficientBase, + retainedWidth, + retainedHeight, + retainedWidth, + -config.Shift2, + normalizeRectangle, + scratch); + } - ref int workspaceBase = ref MemoryMarshal.GetReference(workspace); - ref Av1TransformVector> tempIn = ref Unsafe.As>>(ref workspaceBase); - ref Av1TransformVector> tempOut = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, vectorLength)); - ref Av1TransformVector> step = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, 2 * vectorLength)); - Span buffer = workspace.Slice(Av1TransformWorkspace.Vector512StorageLength, width * height); + /// + /// Loads, flips, and scales one eight-bit residual block into packed transform storage. + /// + private static void LoadPacked( + Span input, + uint inputStride, + ref short destination, + int destinationStride, + int width, + int height, + int shift, + bool flipUpsideDown, + bool flipLeftToRight) + { ref short inputBase = ref MemoryMarshal.GetReference(input); - ref int bufferBase = ref MemoryMarshal.GetReference(buffer); - // Each lane carries one complete column through every stage of the first transform axis. - for (int column = 0; column < width; column += laneCount) + for (int row = 0; row < height; row++) { - for (int row = 0; row < height; row++) + int sourceRow = flipUpsideDown ? height - row - 1 : row; + ref short source = ref Unsafe.Add(ref inputBase, sourceRow * (int)inputStride); + ref short target = ref Unsafe.Add(ref destination, row * destinationStride); + + if (Avx512BW.IsSupported && width >= Vector512.Count) { - int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; - ref short source = ref Unsafe.Add(ref inputBase, (sourceRow * (int)inputStride) + column); - tempIn[row] = Av1Transform2dOperations.RoundShift(Av1Transform2dOperations.Load16Int16(ref source), -shift0); - } + for (int column = 0; column < width; column += Vector512.Count) + { + int sourceColumn = flipLeftToRight ? width - column - Vector512.Count : column; + Vector512 value = Vector512.LoadUnsafe(ref source, (nuint)sourceColumn); + value = flipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + Av1Transform2dOperations.RoundShift(value, -shift).StoreUnsafe(ref target, (nuint)column); + } - TColumnOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitColumn, config.StageRangeColumn); - int destinationColumn = config.FlipLeftToRight ? width - column - laneCount : column; + continue; + } - for (int row = 0; row < height; row++) + if (Avx2.IsSupported && width >= Vector256.Count) { - Vector512 value = Av1Transform2dOperations.RoundShift(tempOut[row], -shift1); - value = config.FlipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; - value.StoreUnsafe(ref bufferBase, (nuint)((row * width) + destinationColumn)); - } - } + for (int column = 0; column < width; column += Vector256.Count) + { + int sourceColumn = flipLeftToRight ? width - column - Vector256.Count : column; + Vector256 value = Vector256.LoadUnsafe(ref source, (nuint)sourceColumn); + value = flipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + Av1Transform2dOperations.RoundShift(value, -shift).StoreUnsafe(ref target, (nuint)column); + } - ref int outputBase = ref MemoryMarshal.GetReference(output); + continue; + } - // Tile transposition changes the lane meaning from columns to rows without scalar gathers. - for (int row = 0; row < height; row += laneCount) - { - for (int column = 0; column < width; column += laneCount) + if (width >= Vector128.Count) { - Vector512 row0 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 0) * width) + column)); - Vector512 row1 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 1) * width) + column)); - Vector512 row2 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 2) * width) + column)); - Vector512 row3 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 3) * width) + column)); - Vector512 row4 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 4) * width) + column)); - Vector512 row5 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 5) * width) + column)); - Vector512 row6 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 6) * width) + column)); - Vector512 row7 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 7) * width) + column)); - Vector512 row8 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 8) * width) + column)); - Vector512 row9 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 9) * width) + column)); - Vector512 row10 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 10) * width) + column)); - Vector512 row11 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 11) * width) + column)); - Vector512 row12 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 12) * width) + column)); - Vector512 row13 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 13) * width) + column)); - Vector512 row14 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 14) * width) + column)); - Vector512 row15 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 15) * width) + column)); - Av1Transform2dOperations.Transpose( - ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, - ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); - - tempIn[column + 0] = row0; - tempIn[column + 1] = row1; - tempIn[column + 2] = row2; - tempIn[column + 3] = row3; - tempIn[column + 4] = row4; - tempIn[column + 5] = row5; - tempIn[column + 6] = row6; - tempIn[column + 7] = row7; - tempIn[column + 8] = row8; - tempIn[column + 9] = row9; - tempIn[column + 10] = row10; - tempIn[column + 11] = row11; - tempIn[column + 12] = row12; - tempIn[column + 13] = row13; - tempIn[column + 14] = row14; - tempIn[column + 15] = row15; - } + for (int column = 0; column < width; column += Vector128.Count) + { + int sourceColumn = flipLeftToRight ? width - column - Vector128.Count : column; + Vector128 value = Vector128.LoadUnsafe(ref source, (nuint)sourceColumn); + value = flipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + Av1Transform2dOperations.RoundShift(value, -shift).StoreUnsafe(ref target, (nuint)column); + } - TRowOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitRow, config.StageRangeRow); + continue; + } - for (int column = 0; column < width; column += laneCount) + // Four-point transforms occupy the lower half of the padded Vector128 row. Loading each source value + // explicitly avoids reading beyond a caller row whose stride is exactly four samples. + for (int column = 0; column < width; column++) { - Vector512 row0 = FinishForward(tempOut[column + 0], -shift2, normalizeRectangle); - Vector512 row1 = FinishForward(tempOut[column + 1], -shift2, normalizeRectangle); - Vector512 row2 = FinishForward(tempOut[column + 2], -shift2, normalizeRectangle); - Vector512 row3 = FinishForward(tempOut[column + 3], -shift2, normalizeRectangle); - Vector512 row4 = FinishForward(tempOut[column + 4], -shift2, normalizeRectangle); - Vector512 row5 = FinishForward(tempOut[column + 5], -shift2, normalizeRectangle); - Vector512 row6 = FinishForward(tempOut[column + 6], -shift2, normalizeRectangle); - Vector512 row7 = FinishForward(tempOut[column + 7], -shift2, normalizeRectangle); - Vector512 row8 = FinishForward(tempOut[column + 8], -shift2, normalizeRectangle); - Vector512 row9 = FinishForward(tempOut[column + 9], -shift2, normalizeRectangle); - Vector512 row10 = FinishForward(tempOut[column + 10], -shift2, normalizeRectangle); - Vector512 row11 = FinishForward(tempOut[column + 11], -shift2, normalizeRectangle); - Vector512 row12 = FinishForward(tempOut[column + 12], -shift2, normalizeRectangle); - Vector512 row13 = FinishForward(tempOut[column + 13], -shift2, normalizeRectangle); - Vector512 row14 = FinishForward(tempOut[column + 14], -shift2, normalizeRectangle); - Vector512 row15 = FinishForward(tempOut[column + 15], -shift2, normalizeRectangle); - Av1Transform2dOperations.Transpose( - ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, - ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); - - row0.StoreUnsafe(ref outputBase, (nuint)(((row + 0) * width) + column)); - row1.StoreUnsafe(ref outputBase, (nuint)(((row + 1) * width) + column)); - row2.StoreUnsafe(ref outputBase, (nuint)(((row + 2) * width) + column)); - row3.StoreUnsafe(ref outputBase, (nuint)(((row + 3) * width) + column)); - row4.StoreUnsafe(ref outputBase, (nuint)(((row + 4) * width) + column)); - row5.StoreUnsafe(ref outputBase, (nuint)(((row + 5) * width) + column)); - row6.StoreUnsafe(ref outputBase, (nuint)(((row + 6) * width) + column)); - row7.StoreUnsafe(ref outputBase, (nuint)(((row + 7) * width) + column)); - row8.StoreUnsafe(ref outputBase, (nuint)(((row + 8) * width) + column)); - row9.StoreUnsafe(ref outputBase, (nuint)(((row + 9) * width) + column)); - row10.StoreUnsafe(ref outputBase, (nuint)(((row + 10) * width) + column)); - row11.StoreUnsafe(ref outputBase, (nuint)(((row + 11) * width) + column)); - row12.StoreUnsafe(ref outputBase, (nuint)(((row + 12) * width) + column)); - row13.StoreUnsafe(ref outputBase, (nuint)(((row + 13) * width) + column)); - row14.StoreUnsafe(ref outputBase, (nuint)(((row + 14) * width) + column)); - row15.StoreUnsafe(ref outputBase, (nuint)(((row + 15) * width) + column)); + int sourceColumn = flipLeftToRight ? width - column - 1 : column; + target = (short)(Unsafe.Add(ref source, sourceColumn) << shift); + target = ref Unsafe.Add(ref target, 1); } } } /// - /// Applies both transform axes with eight samples packed into each SIMD vector. + /// Loads, flips, widens, and scales one residual block into signed thirty-two-bit transform storage. /// - /// The one-dimensional operator applied down each column. - /// The one-dimensional operator applied across each row. - /// The spatial residual samples. - /// The destination transform coefficients. - /// The number of input samples between rows. - /// The transform dimensions, operators, flips, and fixed-point settings. - /// The reusable storage for SIMD vectors and transposed coefficients. - public static void Transform2dVector256( + private static void LoadExpanded( Span input, - Span output, uint inputStride, - ref Av1Transform2dFlipConfiguration config, - Span workspace) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + ref int destination, + int destinationStride, + int width, + int height, + int shift, + bool flipUpsideDown, + bool flipLeftToRight) { - const int laneCount = 8; - const int vectorLength = Av1Constants.MaxTransformSize * laneCount; - - int width = config.TransformSize.GetWidth(); - int height = config.TransformSize.GetHeight(); - int shift0 = config.Shift0; - int shift1 = config.Shift1; - int shift2 = config.Shift2; - bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; - - ref int workspaceBase = ref MemoryMarshal.GetReference(workspace); - ref Av1TransformVector> tempIn = ref Unsafe.As>>(ref workspaceBase); - ref Av1TransformVector> tempOut = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, vectorLength)); - ref Av1TransformVector> step = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, 2 * vectorLength)); - Span buffer = workspace.Slice(Av1TransformWorkspace.Vector256StorageLength, width * height); ref short inputBase = ref MemoryMarshal.GetReference(input); - ref int bufferBase = ref MemoryMarshal.GetReference(buffer); - // Each lane carries one complete column through every stage of the first transform axis. - for (int column = 0; column < width; column += laneCount) + for (int row = 0; row < height; row++) { - for (int row = 0; row < height; row++) + int sourceRow = flipUpsideDown ? height - row - 1 : row; + ref short source = ref Unsafe.Add(ref inputBase, sourceRow * (int)inputStride); + ref int target = ref Unsafe.Add(ref destination, row * destinationStride); + + if (Vector512.IsHardwareAccelerated && width >= Vector512.Count) { - int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; - ref short source = ref Unsafe.Add(ref inputBase, (sourceRow * (int)inputStride) + column); - tempIn[row] = Av1Transform2dOperations.RoundShift(Av1Transform2dOperations.Load8Int16(ref source), -shift0); - } + for (int column = 0; column < width; column += Vector512.Count) + { + int sourceColumn = flipLeftToRight ? width - column - Vector512.Count : column; + Vector512 value = Av1Transform2dOperations.Load16Int16(ref Unsafe.Add(ref source, sourceColumn)); + value = flipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + Av1Transform2dOperations.RoundShift(value, -shift).StoreUnsafe(ref target, (nuint)column); + } - TColumnOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitColumn, config.StageRangeColumn); - int destinationColumn = config.FlipLeftToRight ? width - column - laneCount : column; + continue; + } - for (int row = 0; row < height; row++) + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count) { - Vector256 value = Av1Transform2dOperations.RoundShift(tempOut[row], -shift1); - value = config.FlipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; - value.StoreUnsafe(ref bufferBase, (nuint)((row * width) + destinationColumn)); - } - } + for (int column = 0; column < width; column += Vector256.Count) + { + int sourceColumn = flipLeftToRight ? width - column - Vector256.Count : column; + Vector256 value = Av1Transform2dOperations.Load8Int16(ref Unsafe.Add(ref source, sourceColumn)); + value = flipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + Av1Transform2dOperations.RoundShift(value, -shift).StoreUnsafe(ref target, (nuint)column); + } - ref int outputBase = ref MemoryMarshal.GetReference(output); + continue; + } - // Tile transposition changes the lane meaning from columns to rows without scalar gathers. - for (int row = 0; row < height; row += laneCount) - { - for (int column = 0; column < width; column += laneCount) + if (Vector128.IsHardwareAccelerated) { - Vector256 row0 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 0) * width) + column)); - Vector256 row1 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 1) * width) + column)); - Vector256 row2 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 2) * width) + column)); - Vector256 row3 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 3) * width) + column)); - Vector256 row4 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 4) * width) + column)); - Vector256 row5 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 5) * width) + column)); - Vector256 row6 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 6) * width) + column)); - Vector256 row7 = Vector256.LoadUnsafe(ref bufferBase, (nuint)(((row + 7) * width) + column)); - Av1Transform2dOperations.Transpose(ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7); - tempIn[column + 0] = row0; - tempIn[column + 1] = row1; - tempIn[column + 2] = row2; - tempIn[column + 3] = row3; - tempIn[column + 4] = row4; - tempIn[column + 5] = row5; - tempIn[column + 6] = row6; - tempIn[column + 7] = row7; - } + for (int column = 0; column < width; column += Vector128.Count) + { + int sourceColumn = flipLeftToRight ? width - column - Vector128.Count : column; + Vector128 value = Av1Transform2dOperations.Load4Int16(ref Unsafe.Add(ref source, sourceColumn)); + value = flipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + Av1Transform2dOperations.RoundShift(value, -shift).StoreUnsafe(ref target, (nuint)column); + } - TRowOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitRow, config.StageRangeRow); + continue; + } - for (int column = 0; column < width; column += laneCount) + for (int column = 0; column < width; column++) { - Vector256 row0 = FinishForward(tempOut[column + 0], -shift2, normalizeRectangle); - Vector256 row1 = FinishForward(tempOut[column + 1], -shift2, normalizeRectangle); - Vector256 row2 = FinishForward(tempOut[column + 2], -shift2, normalizeRectangle); - Vector256 row3 = FinishForward(tempOut[column + 3], -shift2, normalizeRectangle); - Vector256 row4 = FinishForward(tempOut[column + 4], -shift2, normalizeRectangle); - Vector256 row5 = FinishForward(tempOut[column + 5], -shift2, normalizeRectangle); - Vector256 row6 = FinishForward(tempOut[column + 6], -shift2, normalizeRectangle); - Vector256 row7 = FinishForward(tempOut[column + 7], -shift2, normalizeRectangle); - Av1Transform2dOperations.Transpose(ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7); - row0.StoreUnsafe(ref outputBase, (nuint)(((row + 0) * width) + column)); - row1.StoreUnsafe(ref outputBase, (nuint)(((row + 1) * width) + column)); - row2.StoreUnsafe(ref outputBase, (nuint)(((row + 2) * width) + column)); - row3.StoreUnsafe(ref outputBase, (nuint)(((row + 3) * width) + column)); - row4.StoreUnsafe(ref outputBase, (nuint)(((row + 4) * width) + column)); - row5.StoreUnsafe(ref outputBase, (nuint)(((row + 5) * width) + column)); - row6.StoreUnsafe(ref outputBase, (nuint)(((row + 6) * width) + column)); - row7.StoreUnsafe(ref outputBase, (nuint)(((row + 7) * width) + column)); + int sourceColumn = flipLeftToRight ? width - column - 1 : column; + target = Unsafe.Add(ref source, sourceColumn) << shift; + target = ref Unsafe.Add(ref target, 1); } } } /// - /// Applies both transform axes with four samples packed into each SIMD vector. + /// Applies one packed transform axis using the widest efficient lane count available for the block. /// - /// The one-dimensional operator applied down each column. - /// The one-dimensional operator applied across each row. - /// The spatial residual samples. - /// The destination transform coefficients. - /// The number of input samples between rows. - /// The transform dimensions, operators, flips, and fixed-point settings. - /// The reusable storage for SIMD vectors and transposed coefficients. - public static void Transform2dVector128( - Span input, - Span output, - uint inputStride, - ref Av1Transform2dFlipConfiguration config, + private static void TransformPackedAxis( + Span buffer, + int transformLength, + int transformCount, + int stride, + int cosBit, Span workspace) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TOperator : struct, IAv1ForwardTransform1dOperator { - const int laneCount = 4; - const int vectorLength = Av1Constants.MaxTransformSize * laneCount; + if (Avx512BW.IsSupported && transformCount >= Vector512.Count) + { + TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + return; + } - int width = config.TransformSize.GetWidth(); - int height = config.TransformSize.GetHeight(); - int shift0 = config.Shift0; - int shift1 = config.Shift1; - int shift2 = config.Shift2; - bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; + if (Avx2.IsSupported && transformCount >= Vector256.Count) + { + TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + return; + } - ref int workspaceBase = ref MemoryMarshal.GetReference(workspace); - ref Av1TransformVector> tempIn = ref Unsafe.As>>(ref workspaceBase); - ref Av1TransformVector> tempOut = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, vectorLength)); - ref Av1TransformVector> step = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, 2 * vectorLength)); - Span buffer = workspace.Slice(Av1TransformWorkspace.Vector128StorageLength, width * height); - ref short inputBase = ref MemoryMarshal.GetReference(input); - ref int bufferBase = ref MemoryMarshal.GetReference(buffer); + TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + } - for (int column = 0; column < width; column += laneCount) + /// + /// Applies one signed thirty-two-bit transform axis using the widest efficient lane count available for the block. + /// + private static void TransformExpandedAxis( + Span buffer, + int transformLength, + int transformCount, + int stride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + if (Vector512.IsHardwareAccelerated && transformCount >= Vector512.Count) { - for (int row = 0; row < height; row++) - { - int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; - ref short source = ref Unsafe.Add(ref inputBase, (sourceRow * (int)inputStride) + column); - tempIn[row] = Av1Transform2dOperations.RoundShift(Av1Transform2dOperations.Load4Int16(ref source), -shift0); - } + TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + return; + } - TColumnOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitColumn, config.StageRangeColumn); - int destinationColumn = config.FlipLeftToRight ? width - column - laneCount : column; + if (Vector256.IsHardwareAccelerated && transformCount >= Vector256.Count) + { + TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + return; + } - for (int row = 0; row < height; row++) - { - Vector128 value = Av1Transform2dOperations.RoundShift(tempOut[row], -shift1); - value = config.FlipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; - value.StoreUnsafe(ref bufferBase, (nuint)((row * width) + destinationColumn)); - } + if (Vector128.IsHardwareAccelerated && transformCount >= Vector128.Count) + { + TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + return; } - ref int outputBase = ref MemoryMarshal.GetReference(output); + TransformAxis(buffer, transformLength, transformCount, stride, cosBit, workspace); + } - for (int row = 0; row < height; row += laneCount) + /// + /// Applies one transform stage network to independent axes held in scalar or SIMD lanes. + /// + private static void TransformAxis( + Span buffer, + int transformLength, + int transformCount, + int stride, + int cosBit, + Span workspace) + where TOperator : struct, IAv1ForwardTransform1dOperator + where TElement : unmanaged + where TValue : struct + { + int vectorByteLength = Unsafe.SizeOf>(); + int laneCount = Unsafe.SizeOf() / Unsafe.SizeOf(); + ref byte workspaceBase = ref Unsafe.As(ref MemoryMarshal.GetReference(workspace)); + ref Av1TransformVector input = ref Unsafe.As>(ref workspaceBase); + ref Av1TransformVector output = ref Unsafe.As>(ref Unsafe.Add(ref workspaceBase, vectorByteLength)); + ref Av1TransformVector step = ref Unsafe.As>(ref Unsafe.Add(ref workspaceBase, 2 * vectorByteLength)); + ref TElement sourceBase = ref MemoryMarshal.GetReference(buffer); + + // Each lane is an independent row or column. Reusing the three caller-owned vectors for every batch keeps + // the complete 1-D stage network in registers without creating a transform-sized stack frame per axis. + for (int batch = 0; batch < transformCount; batch += laneCount) { - for (int column = 0; column < width; column += laneCount) + for (int index = 0; index < transformLength; index++) { - Vector128 row0 = Vector128.LoadUnsafe(ref bufferBase, (nuint)(((row + 0) * width) + column)); - Vector128 row1 = Vector128.LoadUnsafe(ref bufferBase, (nuint)(((row + 1) * width) + column)); - Vector128 row2 = Vector128.LoadUnsafe(ref bufferBase, (nuint)(((row + 2) * width) + column)); - Vector128 row3 = Vector128.LoadUnsafe(ref bufferBase, (nuint)(((row + 3) * width) + column)); - Av1Transform2dOperations.Transpose(ref row0, ref row1, ref row2, ref row3); - tempIn[column + 0] = row0; - tempIn[column + 1] = row1; - tempIn[column + 2] = row2; - tempIn[column + 3] = row3; + ref TElement source = ref Unsafe.Add(ref sourceBase, (index * stride) + batch); + input[index] = Unsafe.ReadUnaligned(ref Unsafe.As(ref source)); } - TRowOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitRow, config.StageRangeRow); + TOperator.Transform(ref input, ref output, ref step, cosBit); - for (int column = 0; column < width; column += laneCount) + for (int index = 0; index < transformLength; index++) { - Vector128 row0 = FinishForward(tempOut[column + 0], -shift2, normalizeRectangle); - Vector128 row1 = FinishForward(tempOut[column + 1], -shift2, normalizeRectangle); - Vector128 row2 = FinishForward(tempOut[column + 2], -shift2, normalizeRectangle); - Vector128 row3 = FinishForward(tempOut[column + 3], -shift2, normalizeRectangle); - Av1Transform2dOperations.Transpose(ref row0, ref row1, ref row2, ref row3); - row0.StoreUnsafe(ref outputBase, (nuint)(((row + 0) * width) + column)); - row1.StoreUnsafe(ref outputBase, (nuint)(((row + 1) * width) + column)); - row2.StoreUnsafe(ref outputBase, (nuint)(((row + 2) * width) + column)); - row3.StoreUnsafe(ref outputBase, (nuint)(((row + 3) * width) + column)); + ref TElement destination = ref Unsafe.Add(ref sourceBase, (index * stride) + batch); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), output[index]); } } } /// - /// Applies both transform axes when hardware vectorization is unavailable. + /// Transposes packed transform storage while applying an AV1 pipeline shift and optional rectangle scaling. /// - /// The one-dimensional operator applied down each column. - /// The one-dimensional operator applied across each row. - /// The spatial residual samples. - /// The destination transform coefficients. - /// The number of input samples between rows. - /// The transform dimensions, operators, flips, and fixed-point settings. - /// The reusable storage for transform stages and transposed coefficients. - public static void Transform2dScalar( - Span input, - Span output, - uint inputStride, - ref Av1Transform2dFlipConfiguration config, - Span workspace) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + private static void TransposePacked( + ref short source, + int sourceStride, + ref short destination, + int destinationStride, + int sourceWidth, + int sourceHeight, + int roundShift, + bool normalizeRectangle, + Span scratch) { - int width = config.TransformSize.GetWidth(); - int height = config.TransformSize.GetHeight(); - int vectorLength = Math.Max(width, height); - int shift0 = config.Shift0; - int shift1 = config.Shift1; - int shift2 = config.Shift2; - bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; - Span tempIn = workspace[..vectorLength]; - Span tempOut = workspace.Slice(vectorLength, vectorLength); - Span step = workspace.Slice(2 * vectorLength, vectorLength); - Span buffer = workspace.Slice(3 * vectorLength, width * height); + int tileSize = Avx2.IsSupported && Math.Min(sourceWidth, sourceHeight) >= 16 ? 16 : Math.Min(sourceWidth, sourceHeight) >= 8 ? 8 : 4; + Span transposeScratch = MemoryMarshal.Cast(scratch); - for (int column = 0; column < width; column++) + for (int row = 0; row < sourceHeight; row += tileSize) { - int inputOffset = config.FlipUpsideDown ? column + ((height - 1) * (int)inputStride) : column; - int inputStep = config.FlipUpsideDown ? -(int)inputStride : (int)inputStride; - - for (int row = 0; row < height; row++) + for (int column = 0; column < sourceWidth; column += tileSize) { - tempIn[row] = input[inputOffset]; - inputOffset += inputStep; - } - - Av1InverseTransformMath.RoundShiftArray(tempIn, height, -shift0); - TColumnOperator.Transform(tempIn, tempOut, step, config.CosBitColumn, config.StageRangeColumn); - Av1InverseTransformMath.RoundShiftArray(tempOut, height, -shift1); - int outputColumn = config.FlipLeftToRight ? width - column - 1 : column; + ref short tileSource = ref Unsafe.Add(ref source, (row * sourceStride) + column); + ref short tileDestination = ref Unsafe.Add(ref destination, (column * destinationStride) + row); - for (int row = 0; row < height; row++) - { - buffer[(row * width) + outputColumn] = tempOut[row]; + if (tileSize == 16) + { + Av1Transform2dOperations.Transpose16x16Int16( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + transposeScratch, + roundShift, + normalizeRectangle); + } + else if (tileSize == 8) + { + Av1Transform2dOperations.Transpose8x8Int16( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + roundShift, + normalizeRectangle); + } + else + { + Av1Transform2dOperations.Transpose4x4Int16( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + roundShift, + normalizeRectangle); + } } } + } - for (int row = 0; row < height; row++) - { - int rowOffset = row * width; - Span outputRow = output.Slice(rowOffset, width); - TRowOperator.Transform(buffer.Slice(rowOffset, width), outputRow, step, config.CosBitRow, config.StageRangeRow); - Av1InverseTransformMath.RoundShiftArray(outputRow, width, -shift2); + /// + /// Promotes and transposes the large packed layouts at the same axis boundary as libaom. + /// + private static void TransposeAndPromote( + ref short source, + int sourceStride, + ref int destination, + int destinationStride, + int sourceWidth, + int sourceHeight, + int roundShift, + Span scratch) + { + int tileSize = Avx512BW.IsSupported ? 16 : Vector256.IsHardwareAccelerated ? 8 : 4; - if (normalizeRectangle) + for (int row = 0; row < sourceHeight; row += tileSize) + { + for (int column = 0; column < sourceWidth; column += tileSize) { - for (int column = 0; column < width; column++) + ref short tileSource = ref Unsafe.Add(ref source, (row * sourceStride) + column); + ref int tileDestination = ref Unsafe.Add(ref destination, (column * destinationStride) + row); + + if (tileSize == 16) + { + Av1Transform2dOperations.Transpose16x16Int16ToInt32( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + scratch[..256], + MemoryMarshal.Cast(scratch[256..]), + roundShift, + false); + } + else if (tileSize == 8) { - outputRow[column] = Av1Math.RoundShift((long)outputRow[column] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + Av1Transform2dOperations.Transpose8x8Int16ToInt32( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + scratch[..64], + roundShift, + false); + } + else + { + Av1Transform2dOperations.Transpose4x4Int16ToInt32( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + scratch[..16], + roundShift, + false); } } } } /// - /// Applies the terminal shift and optional rectangular normalization to four coefficients. + /// Transposes signed thirty-two-bit transform storage while applying the configured terminal operations. /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 FinishForward(Vector128 value, int shift, bool normalizeRectangle) + private static void TransposeExpanded( + ref int source, + int sourceStride, + ref int destination, + int destinationStride, + int sourceWidth, + int sourceHeight, + int roundShift, + bool normalizeRectangle, + Span scratch) { - value = Av1Transform2dOperations.RoundShift(value, shift); - return normalizeRectangle - ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) - : value; - } + bool useVector512 = Vector512.IsHardwareAccelerated && Math.Min(sourceWidth, sourceHeight) >= 16; + int tileSize = useVector512 ? 16 : Vector256.IsHardwareAccelerated && Math.Min(sourceWidth, sourceHeight) >= 8 ? 8 : Vector128.IsHardwareAccelerated ? 4 : 1; + Span transposeScratch = MemoryMarshal.Cast(scratch); - /// - /// Applies the terminal shift and optional rectangular normalization to eight coefficients. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 FinishForward(Vector256 value, int shift, bool normalizeRectangle) - { - value = Av1Transform2dOperations.RoundShift(value, shift); - return normalizeRectangle - ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) - : value; + for (int row = 0; row < sourceHeight; row += tileSize) + { + for (int column = 0; column < sourceWidth; column += tileSize) + { + ref int tileSource = ref Unsafe.Add(ref source, (row * sourceStride) + column); + ref int tileDestination = ref Unsafe.Add(ref destination, (column * destinationStride) + row); + + if (tileSize == 16) + { + Av1Transform2dOperations.Transpose16x16Avx512( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + transposeScratch, + roundShift, + normalizeRectangle); + } + else if (tileSize == 8) + { + Av1Transform2dOperations.Transpose8x8Int32( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + roundShift, + normalizeRectangle); + } + else if (tileSize == 4) + { + Av1Transform2dOperations.Transpose4x4Int32( + ref tileSource, + sourceStride, + ref tileDestination, + destinationStride, + roundShift, + normalizeRectangle); + } + else + { + int value = Av1Math.RoundShift(tileSource, roundShift); + tileDestination = normalizeRectangle + ? Av1Transform1dMath.HalfButterfly(Av1Transform1dMath.NewSqrt2, value, 0, 0, Av1Transform1dMath.NewSqrt2Bits) + : value; + } + } + } } /// - /// Applies the terminal shift and optional rectangular normalization to sixteen coefficients. + /// Widens the completed packed coefficient matrix into its external signed thirty-two-bit representation. /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 FinishForward(Vector512 value, int shift, bool normalizeRectangle) + private static void StorePacked(ref short source, int width, int height, Span destination) { - value = Av1Transform2dOperations.RoundShift(value, shift); - return normalizeRectangle - ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) - : value; + ref int destinationBase = ref MemoryMarshal.GetReference(destination); + + for (int row = 0; row < height; row++) + { + ref short sourceRow = ref Unsafe.Add(ref source, row * width); + ref int destinationRow = ref Unsafe.Add(ref destinationBase, row * width); + int column = 0; + + if (Avx512BW.IsSupported) + { + for (; column <= width - Vector512.Count; column += Vector512.Count) + { + (Vector512 lower, Vector512 upper) = Vector512.Widen(Vector512.LoadUnsafe(ref sourceRow, (nuint)column)); + + lower.StoreUnsafe(ref destinationRow, (nuint)column); + upper.StoreUnsafe(ref destinationRow, (nuint)(column + Vector512.Count)); + } + } + + if (Avx2.IsSupported) + { + for (; column <= width - Vector256.Count; column += Vector256.Count) + { + (Vector256 lower, Vector256 upper) = Vector256.Widen(Vector256.LoadUnsafe(ref sourceRow, (nuint)column)); + + lower.StoreUnsafe(ref destinationRow, (nuint)column); + upper.StoreUnsafe(ref destinationRow, (nuint)(column + Vector256.Count)); + } + } + + for (; column <= width - Vector128.Count; column += Vector128.Count) + { + (Vector128 lower, Vector128 upper) = Vector128.Widen(Vector128.LoadUnsafe(ref sourceRow, (nuint)column)); + + lower.StoreUnsafe(ref destinationRow, (nuint)column); + upper.StoreUnsafe(ref destinationRow, (nuint)(column + Vector128.Count)); + } + + if (column < width) + { + Vector128 value = Vector128.WidenLower(Vector128.Create(Unsafe.As(ref Unsafe.Add(ref sourceRow, column)), 0UL).AsInt16()); + value.StoreUnsafe(ref destinationRow, (nuint)column); + } + } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs index 016bc717a..d17113392 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs @@ -37,18 +37,4 @@ internal readonly struct Av1HighBitDepthInverseTransformOutputOperator : IAv1Inv Vector128 narrowed = Vector128.Narrow(reconstructed.GetLower(), reconstructed.GetUpper()); narrowed.StoreUnsafe(ref destination); } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref short prediction, ref short destination, Vector512 residual, int bitDepth) - { - // AV1 high-bit-depth samples are nonnegative Int16 values. Widening before the residual add preserves signed - // arithmetic, and the bit-depth clamp makes the final narrowing exact for both 10-bit and 12-bit output. - Vector256 packed = Vector256.LoadUnsafe(ref prediction); - (Vector256 predictedLower, Vector256 predictedUpper) = Vector256.Widen(packed); - Vector512 predicted = Vector512.Create(predictedLower, predictedUpper); - Vector512 reconstructed = Vector512.Clamp(predicted + residual, Vector512.Zero, Vector512.Create((1 << bitDepth) - 1)); - Vector256 narrowed = Vector256.Narrow(reconstructed.GetLower(), reconstructed.GetUpper()); - narrowed.StoreUnsafe(ref destination); - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs index ad17cf556..ac2e23b1e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs @@ -238,7 +238,7 @@ internal static class Av1Inverse2dTransformer } /// - /// Applies the specialized operator pair using the widest lane width supported by the block and processor. + /// Applies the specialized operator pair using the production lane width selected for the block and processor. /// private static void Transform2d( Span input, @@ -257,14 +257,6 @@ internal static class Av1Inverse2dTransformer int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); - if (Vector512.IsHardwareAccelerated && width >= Vector512.Count && height >= Vector512.Count) - { - Transform2dVector512( - input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); - - return; - } - if (Vector256.IsHardwareAccelerated && width >= Vector256.Count && height >= Vector256.Count) { Transform2dVector256( @@ -285,168 +277,6 @@ internal static class Av1Inverse2dTransformer input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); } - /// - /// Applies both inverse-transform axes with sixteen samples packed into each SIMD vector. - /// - /// The reconstructed sample storage type. - /// The operator that adds and clips inverse residuals. - /// The one-dimensional operator applied down each column. - /// The one-dimensional operator applied across each row. - /// The dequantized transform coefficients. - /// The prediction samples. - /// The number of prediction samples between rows. - /// The destination reconstruction samples. - /// The number of destination samples between rows. - /// The transform dimensions, operators, flips, and fixed-point settings. - /// The reusable storage for SIMD vectors and transposed coefficients. - /// The coded sample bit depth used to clamp reconstructed values. - public static void Transform2dVector512( - Span input, - Span outputForRead, - int strideForRead, - Span outputForWrite, - int strideForWrite, - ref Av1Transform2dFlipConfiguration config, - Span workspace, - int bitDepth) - where TSample : unmanaged - where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator - { - const int laneCount = 16; - const int vectorLength = Av1Constants.MaxTransformSize * laneCount; - - int width = config.TransformSize.GetWidth(); - int height = config.TransformSize.GetHeight(); - int shift0 = config.Shift0; - int shift1 = config.Shift1; - bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; - byte rowClampBits = (byte)(bitDepth + 8); - byte columnClampBits = (byte)Math.Max(bitDepth + 6, 16); - - ref int workspaceBase = ref MemoryMarshal.GetReference(workspace); - ref Av1TransformVector> tempIn = ref Unsafe.As>>(ref workspaceBase); - ref Av1TransformVector> tempOut = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, vectorLength)); - ref Av1TransformVector> step = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, 2 * vectorLength)); - Span buffer = workspace.Slice(Av1TransformWorkspace.Vector512StorageLength, width * height); - ref int inputBase = ref MemoryMarshal.GetReference(input); - ref int bufferBase = ref MemoryMarshal.GetReference(buffer); - - // Rows are transposed into lanes so the complete 1-D operator runs once for sixteen rows. - for (int row = 0; row < height; row += laneCount) - { - for (int column = 0; column < width; column += laneCount) - { - Vector512 row0 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 0) * width) + column)); - Vector512 row1 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 1) * width) + column)); - Vector512 row2 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 2) * width) + column)); - Vector512 row3 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 3) * width) + column)); - Vector512 row4 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 4) * width) + column)); - Vector512 row5 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 5) * width) + column)); - Vector512 row6 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 6) * width) + column)); - Vector512 row7 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 7) * width) + column)); - Vector512 row8 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 8) * width) + column)); - Vector512 row9 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 9) * width) + column)); - Vector512 row10 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 10) * width) + column)); - Vector512 row11 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 11) * width) + column)); - Vector512 row12 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 12) * width) + column)); - Vector512 row13 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 13) * width) + column)); - Vector512 row14 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 14) * width) + column)); - Vector512 row15 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 15) * width) + column)); - Av1Transform2dOperations.Transpose( - ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, - ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); - - tempIn[column + 0] = PrepareInverseRow(row0, normalizeRectangle, rowClampBits); - tempIn[column + 1] = PrepareInverseRow(row1, normalizeRectangle, rowClampBits); - tempIn[column + 2] = PrepareInverseRow(row2, normalizeRectangle, rowClampBits); - tempIn[column + 3] = PrepareInverseRow(row3, normalizeRectangle, rowClampBits); - tempIn[column + 4] = PrepareInverseRow(row4, normalizeRectangle, rowClampBits); - tempIn[column + 5] = PrepareInverseRow(row5, normalizeRectangle, rowClampBits); - tempIn[column + 6] = PrepareInverseRow(row6, normalizeRectangle, rowClampBits); - tempIn[column + 7] = PrepareInverseRow(row7, normalizeRectangle, rowClampBits); - tempIn[column + 8] = PrepareInverseRow(row8, normalizeRectangle, rowClampBits); - tempIn[column + 9] = PrepareInverseRow(row9, normalizeRectangle, rowClampBits); - tempIn[column + 10] = PrepareInverseRow(row10, normalizeRectangle, rowClampBits); - tempIn[column + 11] = PrepareInverseRow(row11, normalizeRectangle, rowClampBits); - tempIn[column + 12] = PrepareInverseRow(row12, normalizeRectangle, rowClampBits); - tempIn[column + 13] = PrepareInverseRow(row13, normalizeRectangle, rowClampBits); - tempIn[column + 14] = PrepareInverseRow(row14, normalizeRectangle, rowClampBits); - tempIn[column + 15] = PrepareInverseRow(row15, normalizeRectangle, rowClampBits); - } - - TRowOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitRow, config.StageRangeRow); - - for (int column = 0; column < width; column += laneCount) - { - Vector512 row0 = Av1Transform2dOperations.RoundShift(tempOut[column + 0], -shift0); - Vector512 row1 = Av1Transform2dOperations.RoundShift(tempOut[column + 1], -shift0); - Vector512 row2 = Av1Transform2dOperations.RoundShift(tempOut[column + 2], -shift0); - Vector512 row3 = Av1Transform2dOperations.RoundShift(tempOut[column + 3], -shift0); - Vector512 row4 = Av1Transform2dOperations.RoundShift(tempOut[column + 4], -shift0); - Vector512 row5 = Av1Transform2dOperations.RoundShift(tempOut[column + 5], -shift0); - Vector512 row6 = Av1Transform2dOperations.RoundShift(tempOut[column + 6], -shift0); - Vector512 row7 = Av1Transform2dOperations.RoundShift(tempOut[column + 7], -shift0); - Vector512 row8 = Av1Transform2dOperations.RoundShift(tempOut[column + 8], -shift0); - Vector512 row9 = Av1Transform2dOperations.RoundShift(tempOut[column + 9], -shift0); - Vector512 row10 = Av1Transform2dOperations.RoundShift(tempOut[column + 10], -shift0); - Vector512 row11 = Av1Transform2dOperations.RoundShift(tempOut[column + 11], -shift0); - Vector512 row12 = Av1Transform2dOperations.RoundShift(tempOut[column + 12], -shift0); - Vector512 row13 = Av1Transform2dOperations.RoundShift(tempOut[column + 13], -shift0); - Vector512 row14 = Av1Transform2dOperations.RoundShift(tempOut[column + 14], -shift0); - Vector512 row15 = Av1Transform2dOperations.RoundShift(tempOut[column + 15], -shift0); - Av1Transform2dOperations.Transpose( - ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, - ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); - - row0.StoreUnsafe(ref bufferBase, (nuint)(((row + 0) * width) + column)); - row1.StoreUnsafe(ref bufferBase, (nuint)(((row + 1) * width) + column)); - row2.StoreUnsafe(ref bufferBase, (nuint)(((row + 2) * width) + column)); - row3.StoreUnsafe(ref bufferBase, (nuint)(((row + 3) * width) + column)); - row4.StoreUnsafe(ref bufferBase, (nuint)(((row + 4) * width) + column)); - row5.StoreUnsafe(ref bufferBase, (nuint)(((row + 5) * width) + column)); - row6.StoreUnsafe(ref bufferBase, (nuint)(((row + 6) * width) + column)); - row7.StoreUnsafe(ref bufferBase, (nuint)(((row + 7) * width) + column)); - row8.StoreUnsafe(ref bufferBase, (nuint)(((row + 8) * width) + column)); - row9.StoreUnsafe(ref bufferBase, (nuint)(((row + 9) * width) + column)); - row10.StoreUnsafe(ref bufferBase, (nuint)(((row + 10) * width) + column)); - row11.StoreUnsafe(ref bufferBase, (nuint)(((row + 11) * width) + column)); - row12.StoreUnsafe(ref bufferBase, (nuint)(((row + 12) * width) + column)); - row13.StoreUnsafe(ref bufferBase, (nuint)(((row + 13) * width) + column)); - row14.StoreUnsafe(ref bufferBase, (nuint)(((row + 14) * width) + column)); - row15.StoreUnsafe(ref bufferBase, (nuint)(((row + 15) * width) + column)); - } - } - - ref TSample readBase = ref MemoryMarshal.GetReference(outputForRead); - ref TSample writeBase = ref MemoryMarshal.GetReference(outputForWrite); - - // The intermediate rows already contain contiguous column groups, avoiding a second transpose. - for (int column = 0; column < width; column += laneCount) - { - int sourceColumn = config.FlipLeftToRight ? width - column - laneCount : column; - - for (int row = 0; row < height; row++) - { - Vector512 value = Vector512.LoadUnsafe(ref bufferBase, (nuint)((row * width) + sourceColumn)); - value = config.FlipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; - tempIn[row] = Av1Transform1dMath.Clamp(value, columnClampBits); - } - - TColumnOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitColumn, config.StageRangeColumn); - - for (int row = 0; row < height; row++) - { - int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; - Vector512 residual = Av1Transform2dOperations.RoundShift(tempOut[sourceRow], -shift1); - ref TSample prediction = ref Unsafe.Add(ref readBase, (row * strideForRead) + column); - ref TSample destination = ref Unsafe.Add(ref writeBase, (row * strideForWrite) + column); - TOutputOperator.Add(ref prediction, ref destination, residual, bitDepth); - } - } - } - /// /// Applies both inverse-transform axes with eight samples packed into each SIMD vector. /// @@ -481,6 +311,9 @@ internal static class Av1Inverse2dTransformer int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); + Av1TransformSize adjustedTransformSize = config.TransformSize.GetAdjusted(); + int inputWidth = adjustedTransformSize.GetWidth(); + int inputHeight = adjustedTransformSize.GetHeight(); int shift0 = config.Shift0; int shift1 = config.Shift1; bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; @@ -500,14 +333,39 @@ internal static class Av1Inverse2dTransformer { for (int column = 0; column < width; column += laneCount) { - Vector256 row0 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 0) * width) + column)); - Vector256 row1 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 1) * width) + column)); - Vector256 row2 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 2) * width) + column)); - Vector256 row3 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 3) * width) + column)); - Vector256 row4 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 4) * width) + column)); - Vector256 row5 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 5) * width) + column)); - Vector256 row6 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 6) * width) + column)); - Vector256 row7 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 7) * width) + column)); + bool hasCodedCoefficients = row < inputHeight && column < inputWidth; + Vector256 row0; + Vector256 row1; + Vector256 row2; + Vector256 row3; + Vector256 row4; + Vector256 row5; + Vector256 row6; + Vector256 row7; + + if (hasCodedCoefficients) + { + row0 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 0) * inputWidth) + column)); + row1 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 1) * inputWidth) + column)); + row2 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 2) * inputWidth) + column)); + row3 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 3) * inputWidth) + column)); + row4 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 4) * inputWidth) + column)); + row5 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 5) * inputWidth) + column)); + row6 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 6) * inputWidth) + column)); + row7 = Vector256.LoadUnsafe(ref inputBase, (nuint)(((row + 7) * inputWidth) + column)); + } + else + { + row0 = Vector256.Zero; + row1 = Vector256.Zero; + row2 = Vector256.Zero; + row3 = Vector256.Zero; + row4 = Vector256.Zero; + row5 = Vector256.Zero; + row6 = Vector256.Zero; + row7 = Vector256.Zero; + } + Av1Transform2dOperations.Transpose(ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7); tempIn[column + 0] = PrepareInverseRow(row0, normalizeRectangle, rowClampBits); tempIn[column + 1] = PrepareInverseRow(row1, normalizeRectangle, rowClampBits); @@ -605,6 +463,9 @@ internal static class Av1Inverse2dTransformer int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); + Av1TransformSize adjustedTransformSize = config.TransformSize.GetAdjusted(); + int inputWidth = adjustedTransformSize.GetWidth(); + int inputHeight = adjustedTransformSize.GetHeight(); int shift0 = config.Shift0; int shift1 = config.Shift1; bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; @@ -623,10 +484,27 @@ internal static class Av1Inverse2dTransformer { for (int column = 0; column < width; column += laneCount) { - Vector128 row0 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 0) * width) + column)); - Vector128 row1 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 1) * width) + column)); - Vector128 row2 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 2) * width) + column)); - Vector128 row3 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 3) * width) + column)); + bool hasCodedCoefficients = row < inputHeight && column < inputWidth; + Vector128 row0; + Vector128 row1; + Vector128 row2; + Vector128 row3; + + if (hasCodedCoefficients) + { + row0 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 0) * inputWidth) + column)); + row1 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 1) * inputWidth) + column)); + row2 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 2) * inputWidth) + column)); + row3 = Vector128.LoadUnsafe(ref inputBase, (nuint)(((row + 3) * inputWidth) + column)); + } + else + { + row0 = Vector128.Zero; + row1 = Vector128.Zero; + row2 = Vector128.Zero; + row3 = Vector128.Zero; + } + Av1Transform2dOperations.Transpose(ref row0, ref row1, ref row2, ref row3); tempIn[column + 0] = PrepareInverseRow(row0, normalizeRectangle, rowClampBits); tempIn[column + 1] = PrepareInverseRow(row1, normalizeRectangle, rowClampBits); @@ -708,6 +586,9 @@ internal static class Av1Inverse2dTransformer { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); + Av1TransformSize adjustedTransformSize = config.TransformSize.GetAdjusted(); + int inputWidth = adjustedTransformSize.GetWidth(); + int inputHeight = adjustedTransformSize.GetHeight(); int vectorLength = Math.Max(width, height); int shift0 = config.Shift0; int shift1 = config.Shift1; @@ -722,14 +603,20 @@ internal static class Av1Inverse2dTransformer for (int row = 0; row < height; row++) { int rowOffset = row * width; + tempIn[..width].Clear(); - for (int column = 0; column < width; column++) + if (row < inputHeight) { - int value = input[rowOffset + column]; - value = normalizeRectangle - ? Av1Math.RoundShift((long)value * Av1InverseTransformMath.NewInverseSqrt2, Av1InverseTransformMath.NewSqrt2BitCount) - : value; - tempIn[column] = Av1Transform1dMath.Clamp(value, rowClampBits); + int inputOffset = row * inputWidth; + + for (int column = 0; column < inputWidth; column++) + { + int value = input[inputOffset + column]; + value = normalizeRectangle + ? Av1Math.RoundShift((long)value * Av1InverseTransformMath.NewInverseSqrt2, Av1InverseTransformMath.NewSqrt2BitCount) + : value; + tempIn[column] = Av1Transform1dMath.Clamp(value, rowClampBits); + } } TRowOperator.Transform(tempIn, tempOut, step, config.CosBitRow, config.StageRangeRow); @@ -784,17 +671,4 @@ internal static class Av1Inverse2dTransformer return Av1Transform1dMath.Clamp(value, clampBits); } - - /// - /// Applies rectangular normalization and the row-input clamp to sixteen coefficient lanes. - /// - private static Vector512 PrepareInverseRow(Vector512 value, bool normalizeRectangle, byte clampBits) - { - if (normalizeRectangle) - { - value = Av1Transform1dMath.MultiplyRound(value, Av1InverseTransformMath.NewInverseSqrt2, Av1InverseTransformMath.NewSqrt2BitCount); - } - - return Av1Transform1dMath.Clamp(value, clampBits); - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs index d1ea5f9db..a478abdf1 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -107,6 +108,214 @@ internal static class Av1Transform1dMath return (weightedSum + Vector512.Create(1 << (cosBit - 1))) >> cosBit; } + /// + /// Adds and subtracts thirty-two pairs of low-bit-depth transform values with signed saturation. + /// + /// The first thirty-two input values. + /// The second thirty-two input values. + /// The thirty-two saturated sums. + /// The thirty-two saturated differences. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void AddSubtract( + in Vector512 input0, + in Vector512 input1, + out Vector512 sum, + out Vector512 difference) + { + // Read both operands before either destination is written because libaom deliberately permits an input + // buffer to alias one or both outputs while alternating between its two fixed transform-stage buffers. + Vector512 left = input0; + Vector512 right = input1; + + sum = Vector512.AddSaturate(left, right); + difference = Vector512.SubtractSaturate(left, right); + } + + /// + /// Calculates both outputs of thirty-two rounded low-bit-depth butterflies in parallel. + /// + /// The first fixed-point weight. + /// The second fixed-point weight. + /// The first thirty-two input values. + /// The second thirty-two input values. + /// The first thirty-two saturated, rounded results. + /// The second thirty-two saturated, rounded results. + /// The number of fractional bits in each weight. + /// The rounding offset for the widened intermediate values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Butterfly( + int weight0, + int weight1, + in Vector512 input0, + in Vector512 input1, + out Vector512 output0, + out Vector512 output1, + int cosBit, + in Vector512 rounding) + { + // VPMADDWD evaluates adjacent Int16 products into Int32 lanes. Both outputs reuse the same interleaved + // inputs, matching libaom's whole butterfly instead of loading and unpacking each input pair twice. + Vector512 left = input0; + Vector512 right = input1; + Vector512 interleavedLower = Avx512BW.UnpackLow(left, right); + Vector512 interleavedUpper = Avx512BW.UnpackHigh(left, right); + Vector512 weight0Values = Vector512.Create((short)weight0); + Vector512 weight1Values = Vector512.Create((short)weight1); + Vector512 weights0 = Avx512BW.UnpackLow(weight0Values, weight1Values); + Vector512 weights1 = Avx512BW.UnpackLow(weight1Values, Vector512.Create((short)-weight0)); + Vector512 output0Lower = Avx512BW.MultiplyAddAdjacent(interleavedLower, weights0); + Vector512 output0Upper = Avx512BW.MultiplyAddAdjacent(interleavedUpper, weights0); + Vector512 output1Lower = Avx512BW.MultiplyAddAdjacent(interleavedLower, weights1); + Vector512 output1Upper = Avx512BW.MultiplyAddAdjacent(interleavedUpper, weights1); + + output0Lower = (output0Lower + rounding) >> cosBit; + output0Upper = (output0Upper + rounding) >> cosBit; + output1Lower = (output1Lower + rounding) >> cosBit; + output1Upper = (output1Upper + rounding) >> cosBit; + + // VPACKSSDW restores the original lane order within each 128-bit block and narrows with the saturation + // required by the low-bit-depth AV1 stage arithmetic. + output0 = Avx512BW.PackSignedSaturate(output0Lower, output0Upper); + output1 = Avx512BW.PackSignedSaturate(output1Lower, output1Upper); + } + + /// + /// Adds and subtracts sixteen pairs of low-bit-depth transform values with signed saturation. + /// + /// The first sixteen input values. + /// The second sixteen input values. + /// The sixteen saturated sums. + /// The sixteen saturated differences. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void AddSubtract( + in Vector256 input0, + in Vector256 input1, + out Vector256 sum, + out Vector256 difference) + { + Vector256 left = input0; + Vector256 right = input1; + + sum = Vector256.AddSaturate(left, right); + difference = Vector256.SubtractSaturate(left, right); + } + + /// + /// Calculates both outputs of sixteen rounded low-bit-depth butterflies in parallel. + /// + /// The first fixed-point weight. + /// The second fixed-point weight. + /// The first sixteen input values. + /// The second sixteen input values. + /// The first sixteen saturated, rounded results. + /// The second sixteen saturated, rounded results. + /// The number of fractional bits in each weight. + /// The rounding offset for the widened intermediate values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Butterfly( + int weight0, + int weight1, + in Vector256 input0, + in Vector256 input1, + out Vector256 output0, + out Vector256 output1, + int cosBit, + in Vector256 rounding) + { + Vector256 left = input0; + Vector256 right = input1; + Vector256 interleavedLower = Avx2.UnpackLow(left, right); + Vector256 interleavedUpper = Avx2.UnpackHigh(left, right); + Vector256 weight0Values = Vector256.Create((short)weight0); + Vector256 weight1Values = Vector256.Create((short)weight1); + Vector256 weights0 = Avx2.UnpackLow(weight0Values, weight1Values); + Vector256 weights1 = Avx2.UnpackLow(weight1Values, Vector256.Create((short)-weight0)); + Vector256 output0Lower = Avx2.MultiplyAddAdjacent(interleavedLower, weights0); + Vector256 output0Upper = Avx2.MultiplyAddAdjacent(interleavedUpper, weights0); + Vector256 output1Lower = Avx2.MultiplyAddAdjacent(interleavedLower, weights1); + Vector256 output1Upper = Avx2.MultiplyAddAdjacent(interleavedUpper, weights1); + + output0Lower = (output0Lower + rounding) >> cosBit; + output0Upper = (output0Upper + rounding) >> cosBit; + output1Lower = (output1Lower + rounding) >> cosBit; + output1Upper = (output1Upper + rounding) >> cosBit; + + output0 = Avx2.PackSignedSaturate(output0Lower, output0Upper); + output1 = Avx2.PackSignedSaturate(output1Lower, output1Upper); + } + + /// + /// Calculates both outputs of eight rounded low-bit-depth butterflies in parallel. + /// + /// The first fixed-point weight. + /// The second fixed-point weight. + /// The first eight input values. + /// The second eight input values. + /// The first eight saturated, rounded results. + /// The second eight saturated, rounded results. + /// The number of fractional bits in each weight. + /// The rounding offset for the widened intermediate values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Butterfly( + int weight0, + int weight1, + in Vector128 input0, + in Vector128 input1, + out Vector128 output0, + out Vector128 output1, + int cosBit, + in Vector128 rounding) + { + Vector128 left = input0; + Vector128 right = input1; + + if (Sse2.IsSupported) + { + // PMADDWD is the native x86 form of Highway's pairwise widening multiply-add. Interleaving once lets + // both butterfly outputs reuse the same input arrangement before signed-saturating demotion. + Vector128 interleavedLower = Sse2.UnpackLow(left, right); + Vector128 interleavedUpper = Sse2.UnpackHigh(left, right); + Vector128 weight0Values = Vector128.Create((short)weight0); + Vector128 weight1Values = Vector128.Create((short)weight1); + Vector128 weights0 = Sse2.UnpackLow(weight0Values, weight1Values); + Vector128 weights1 = Sse2.UnpackLow(weight1Values, Vector128.Create((short)-weight0)); + Vector128 output0Lower = Sse2.MultiplyAddAdjacent(interleavedLower, weights0); + Vector128 output0Upper = Sse2.MultiplyAddAdjacent(interleavedUpper, weights0); + Vector128 output1Lower = Sse2.MultiplyAddAdjacent(interleavedLower, weights1); + Vector128 output1Upper = Sse2.MultiplyAddAdjacent(interleavedUpper, weights1); + + output0Lower = (output0Lower + rounding) >> cosBit; + output0Upper = (output0Upper + rounding) >> cosBit; + output1Lower = (output1Lower + rounding) >> cosBit; + output1Upper = (output1Upper + rounding) >> cosBit; + + output0 = Sse2.PackSignedSaturate(output0Lower, output0Upper); + output1 = Sse2.PackSignedSaturate(output1Lower, output1Upper); + return; + } + + // AdvSimd and WebAssembly do not expose PMADDWD. Widen both inputs once and retain the complete operation + // in Vector128 lanes so those targets still execute the transform as a whole SIMD butterfly. + (Vector128 leftLower, Vector128 leftUpper) = Vector128.Widen(left); + (Vector128 rightLower, Vector128 rightUpper) = Vector128.Widen(right); + + Vector128 weight0Vector = Vector128.Create(weight0); + Vector128 weight1Vector = Vector128.Create(weight1); + Vector128 output0LowerVector = ((leftLower * weight0Vector) + (rightLower * weight1Vector) + rounding) >> cosBit; + Vector128 output0UpperVector = ((leftUpper * weight0Vector) + (rightUpper * weight1Vector) + rounding) >> cosBit; + Vector128 output1LowerVector = ((leftLower * weight1Vector) - (rightLower * weight0Vector) + rounding) >> cosBit; + Vector128 output1UpperVector = ((leftUpper * weight1Vector) - (rightUpper * weight0Vector) + rounding) >> cosBit; + Vector128 minimum = Vector128.Create((int)short.MinValue); + Vector128 maximum = Vector128.Create((int)short.MaxValue); + + output0LowerVector = Vector128.Clamp(output0LowerVector, minimum, maximum); + output0UpperVector = Vector128.Clamp(output0UpperVector, minimum, maximum); + output1LowerVector = Vector128.Clamp(output1LowerVector, minimum, maximum); + output1UpperVector = Vector128.Clamp(output1UpperVector, minimum, maximum); + output0 = Vector128.Narrow(output0LowerVector, output0UpperVector); + output1 = Vector128.Narrow(output1LowerVector, output1UpperVector); + } + /// /// Clamps four transform-stage values to the signed range represented by a bit count. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs index d2de5398b..f45a80354 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs @@ -2,7 +2,9 @@ // Licensed under the Six Labors Split License. using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -12,6 +14,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// internal static class Av1Transform2dOperations { + /// + /// Gets the row order produced by the final AVX-512 16-by-16 transpose concatenation. + /// + private static ReadOnlySpan Vector512TransposeStoreOrder => [0, 2, 1, 3, 4, 6, 5, 7, 8, 10, 9, 11, 12, 14, 13, 15]; + + /// + /// Gets the row order produced by the final sixteen-bit 16-by-16 transpose concatenation. + /// + private static ReadOnlySpan Int16TransposeStoreOrder => [0, 4, 2, 6, 1, 5, 3, 7, 8, 12, 10, 14, 9, 13, 11, 15]; + /// /// Loads four signed sixteen-bit values and widens them to four signed thirty-two-bit lanes. /// @@ -97,6 +109,354 @@ internal static class Av1Transform2dOperations return bit < 0 ? value << -bit : value; } + /// + /// Applies a signed AV1 pipeline shift to eight signed sixteen-bit values in parallel. + /// + /// The values to shift. + /// A positive rounded-right shift or a negative exact-left shift. + /// The shifted values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 RoundShift(Vector128 value, int bit) + { + if (bit > 0) + { + return (value + Vector128.Create((short)(1 << (bit - 1)))) >> bit; + } + + return bit < 0 ? value << -bit : value; + } + + /// + /// Applies a signed AV1 pipeline shift to sixteen signed sixteen-bit values in parallel. + /// + /// The values to shift. + /// A positive rounded-right shift or a negative exact-left shift. + /// The shifted values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 RoundShift(Vector256 value, int bit) + { + if (bit > 0) + { + // Conformant low-bit-depth stage ranges leave room for the rounding bias, so this intentionally uses the + // wrapping add used by libaom rather than changing the normative result with a saturating instruction. + return (value + Vector256.Create((short)(1 << (bit - 1)))) >> bit; + } + + return bit < 0 ? value << -bit : value; + } + + /// + /// Applies a signed AV1 pipeline shift to thirty-two signed sixteen-bit values in parallel. + /// + /// The values to shift. + /// A positive rounded-right shift or a negative exact-left shift. + /// The shifted values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 RoundShift(Vector512 value, int bit) + { + if (bit > 0) + { + return (value + Vector512.Create((short)(1 << (bit - 1)))) >> bit; + } + + return bit < 0 ? value << -bit : value; + } + + /// + /// Transposes one 4-by-4 tile of signed sixteen-bit values and applies the configured pipeline operations. + /// + /// The first value of the source tile. + /// The number of signed sixteen-bit values between source rows. + /// The first value of the destination tile. + /// The number of signed sixteen-bit values between destination rows. + /// The signed AV1 pipeline shift applied before transposition. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose4x4Int16( + ref short source, + int sourceStride, + ref short destination, + int destinationStride, + int roundShift, + bool normalizeRectangle) + { + Vector128 row0 = Load4Short(ref source); + Vector128 row1 = Load4Short(ref Unsafe.Add(ref source, sourceStride)); + Vector128 row2 = Load4Short(ref Unsafe.Add(ref source, 2 * sourceStride)); + Vector128 row3 = Load4Short(ref Unsafe.Add(ref source, 3 * sourceStride)); + + row0 = Finish(row0, roundShift, normalizeRectangle); + row1 = Finish(row1, roundShift, normalizeRectangle); + row2 = Finish(row2, roundShift, normalizeRectangle); + row3 = Finish(row3, roundShift, normalizeRectangle); + + // Only the lower four lanes belong to the tile. Interleaving at Int16 and Int32 granularity exchanges the + // two row-index bits with the corresponding column-index bits without touching adjacent padded storage. + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair1 = Vector128_.UnpackLow(row2, row3); + Vector128 columns01 = Vector128_.UnpackLow(pair0.AsInt32(), pair1.AsInt32()); + Vector128 columns23 = Vector128_.UnpackHigh(pair0.AsInt32(), pair1.AsInt32()); + + Store4Int16(columns01.AsUInt64().GetElement(0), ref destination); + Store4Int16(columns01.AsUInt64().GetElement(1), ref Unsafe.Add(ref destination, destinationStride)); + Store4Int16(columns23.AsUInt64().GetElement(0), ref Unsafe.Add(ref destination, 2 * destinationStride)); + Store4Int16(columns23.AsUInt64().GetElement(1), ref Unsafe.Add(ref destination, 3 * destinationStride)); + } + + /// + /// Transposes one 8-by-8 tile of signed sixteen-bit values and applies the configured pipeline operations. + /// + /// The first value of the source tile. + /// The number of signed sixteen-bit values between source rows. + /// The first value of the destination tile. + /// The number of signed sixteen-bit values between destination rows. + /// The signed AV1 pipeline shift applied before transposition. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose8x8Int16( + ref short source, + int sourceStride, + ref short destination, + int destinationStride, + int roundShift, + bool normalizeRectangle) + { + Vector128 row0 = Finish(Vector128.LoadUnsafe(ref source), roundShift, normalizeRectangle); + Vector128 row1 = Finish(Vector128.LoadUnsafe(ref source, (nuint)sourceStride), roundShift, normalizeRectangle); + Vector128 row2 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(2 * sourceStride)), roundShift, normalizeRectangle); + Vector128 row3 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(3 * sourceStride)), roundShift, normalizeRectangle); + Vector128 row4 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(4 * sourceStride)), roundShift, normalizeRectangle); + Vector128 row5 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(5 * sourceStride)), roundShift, normalizeRectangle); + Vector128 row6 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(6 * sourceStride)), roundShift, normalizeRectangle); + Vector128 row7 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(7 * sourceStride)), roundShift, normalizeRectangle); + + Vector128 pair0 = Vector128_.UnpackLow(row0, row1); + Vector128 pair1 = Vector128_.UnpackHigh(row0, row1); + Vector128 pair2 = Vector128_.UnpackLow(row2, row3); + Vector128 pair3 = Vector128_.UnpackHigh(row2, row3); + Vector128 pair4 = Vector128_.UnpackLow(row4, row5); + Vector128 pair5 = Vector128_.UnpackHigh(row4, row5); + Vector128 pair6 = Vector128_.UnpackLow(row6, row7); + Vector128 pair7 = Vector128_.UnpackHigh(row6, row7); + Vector128 quad0 = Vector128_.UnpackLow(pair0.AsInt32(), pair2.AsInt32()); + Vector128 quad1 = Vector128_.UnpackHigh(pair0.AsInt32(), pair2.AsInt32()); + Vector128 quad2 = Vector128_.UnpackLow(pair1.AsInt32(), pair3.AsInt32()); + Vector128 quad3 = Vector128_.UnpackHigh(pair1.AsInt32(), pair3.AsInt32()); + Vector128 quad4 = Vector128_.UnpackLow(pair4.AsInt32(), pair6.AsInt32()); + Vector128 quad5 = Vector128_.UnpackHigh(pair4.AsInt32(), pair6.AsInt32()); + Vector128 quad6 = Vector128_.UnpackLow(pair5.AsInt32(), pair7.AsInt32()); + Vector128 quad7 = Vector128_.UnpackHigh(pair5.AsInt32(), pair7.AsInt32()); + + Vector128_.UnpackLow(quad0.AsInt64(), quad4.AsInt64()).AsInt16().StoreUnsafe(ref destination); + Vector128_.UnpackHigh(quad0.AsInt64(), quad4.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)destinationStride); + Vector128_.UnpackLow(quad1.AsInt64(), quad5.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)(2 * destinationStride)); + Vector128_.UnpackHigh(quad1.AsInt64(), quad5.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)(3 * destinationStride)); + Vector128_.UnpackLow(quad2.AsInt64(), quad6.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)(4 * destinationStride)); + Vector128_.UnpackHigh(quad2.AsInt64(), quad6.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)(5 * destinationStride)); + Vector128_.UnpackLow(quad3.AsInt64(), quad7.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)(6 * destinationStride)); + Vector128_.UnpackHigh(quad3.AsInt64(), quad7.AsInt64()).AsInt16().StoreUnsafe(ref destination, (nuint)(7 * destinationStride)); + } + + /// + /// Transposes one 16-by-16 tile of signed sixteen-bit values and applies the configured pipeline shift. + /// + /// The first value of the source tile. + /// The number of signed sixteen-bit values between source rows. + /// The first value of the destination tile. + /// The number of signed sixteen-bit values between destination rows. + /// The reusable storage for the widening transpose stages. + /// The signed AV1 pipeline shift applied before transposition. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose16x16Int16( + ref short source, + int sourceStride, + ref short destination, + int destinationStride, + Span scratch, + int roundShift, + bool normalizeRectangle) + { + ref long scratch64 = ref MemoryMarshal.GetReference(scratch); + ref int scratch32 = ref Unsafe.As(ref scratch64); + + // Pairing adjacent rows widens groups of two Int16 values into Int32 storage. The widening is a bitwise + // reinterpretation: it preserves all sixteen source bits while progressively exchanging row and column bits. + for (int row = 0; row < 16; row += 2) + { + Vector256 even = Vector256.LoadUnsafe(ref source, (nuint)(row * sourceStride)); + Vector256 odd = Vector256.LoadUnsafe(ref source, (nuint)((row + 1) * sourceStride)); + even = RoundShift(even, roundShift); + odd = RoundShift(odd, roundShift); + + if (normalizeRectangle) + { + even = Forward.Av1ForwardTransformArithmetic>.MultiplyRound( + even, + Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + + odd = Forward.Av1ForwardTransformArithmetic>.MultiplyRound( + odd, + Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits); + } + + Avx2.UnpackLow(even, odd).AsInt32().StoreUnsafe(ref scratch32, (nuint)(row * 8)); + Avx2.UnpackHigh(even, odd).AsInt32().StoreUnsafe(ref scratch32, (nuint)((row + 1) * 8)); + } + + // The Int32 and Int64 views exchange the next two index bits without allocating another temporary buffer. + // Each group is fully consumed before its destination slots overwrite the same scratch locations. + for (int row = 0; row < 16; row += 4) + { + for (int offset = 0; offset < 2; offset++) + { + Vector256 lower = Vector256.LoadUnsafe(ref scratch32, (nuint)((row + offset) * 8)); + Vector256 upper = Vector256.LoadUnsafe(ref scratch32, (nuint)((row + offset + 2) * 8)); + Avx2.UnpackLow(lower, upper).AsInt64().StoreUnsafe(ref scratch64, (nuint)((row + offset) * 4)); + Avx2.UnpackHigh(lower, upper).AsInt64().StoreUnsafe(ref scratch64, (nuint)((row + offset + 2) * 4)); + } + } + + for (int row = 0; row < 16; row += 8) + { + for (int offset = 0; offset < 4; offset++) + { + Vector256 lower = Vector256.LoadUnsafe(ref scratch64, (nuint)((row + offset) * 4)); + Vector256 upper = Vector256.LoadUnsafe(ref scratch64, (nuint)((row + offset + 4) * 4)); + Avx2.UnpackLow(lower, upper).StoreUnsafe(ref scratch64, (nuint)((row + offset) * 4)); + Avx2.UnpackHigh(lower, upper).StoreUnsafe(ref scratch64, (nuint)((row + offset + 4) * 4)); + } + } + + // Concatenating the matching 128-bit halves restores sixteen Int16 lanes per output row. The staged unpack + // order produces a fixed row permutation, so the compile-time table maps each register to its true column. + for (int row = 0; row < 8; row++) + { + Vector256 lower = Vector256.LoadUnsafe(ref scratch64, (nuint)(row * 4)); + Vector256 upper = Vector256.LoadUnsafe(ref scratch64, (nuint)((row + 8) * 4)); + Vector256 lowerResult = Vector256.Create(lower.GetLower(), upper.GetLower()).AsInt16(); + Vector256 upperResult = Vector256.Create(lower.GetUpper(), upper.GetUpper()).AsInt16(); + int lowerDestinationRow = Int16TransposeStoreOrder[row]; + int upperDestinationRow = Int16TransposeStoreOrder[row + 8]; + lowerResult.StoreUnsafe(ref destination, (nuint)(lowerDestinationRow * destinationStride)); + upperResult.StoreUnsafe(ref destination, (nuint)(upperDestinationRow * destinationStride)); + } + } + + /// + /// Promotes and transposes one 16-by-16 tile of signed sixteen-bit values. + /// + /// The first value of the source tile. + /// The number of signed sixteen-bit values between source rows. + /// The first value of the destination tile. + /// The number of signed thirty-two-bit values between destination rows. + /// The reusable storage for the promoted source tile. + /// The reusable storage for the widening transpose stages. + /// The signed AV1 pipeline shift applied after promotion. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose16x16Int16ToInt32( + ref short source, + int sourceStride, + ref int destination, + int destinationStride, + Span promotionBuffer, + Span transposeScratch, + int roundShift, + bool normalizeRectangle) + { + ref int promotionBase = ref MemoryMarshal.GetReference(promotionBuffer); + + // The large low-bit-depth transforms widen at the axis boundary. Applying the pipeline shift after widening + // is significant: a left shift that is valid in Int32 is not required to remain representable in Int16. + for (int row = 0; row < 16; row++) + { + Vector256 packed = Vector256.LoadUnsafe(ref source, (nuint)(row * sourceStride)); + (Vector256 lower, Vector256 upper) = Vector256.Widen(packed); + + Vector512.Create(lower, upper).StoreUnsafe(ref promotionBase, (nuint)(row * 16)); + } + + // Once promoted, the same bounded transpose used by the high-bit-depth AVX-512 path supplies the exact + // libaom staging order and performs the axis-boundary shift in signed thirty-two-bit lanes. + Transpose16x16Avx512( + ref promotionBase, + 16, + ref destination, + destinationStride, + transposeScratch, + roundShift, + normalizeRectangle); + } + + /// + /// Promotes and transposes one 8-by-8 tile of signed sixteen-bit values. + /// + /// The first value of the source tile. + /// The number of signed sixteen-bit values between source rows. + /// The first value of the destination tile. + /// The number of signed thirty-two-bit values between destination rows. + /// The reusable storage for the promoted source tile. + /// The signed AV1 pipeline shift applied after promotion. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose8x8Int16ToInt32( + ref short source, + int sourceStride, + ref int destination, + int destinationStride, + Span promotionBuffer, + int roundShift, + bool normalizeRectangle) + { + ref int promotionBase = ref MemoryMarshal.GetReference(promotionBuffer); + + // AVX2 processes eight Int32 transform axes at a time. Widening each packed row before the axis shift follows + // libaom's Repartition boundary and prevents valid Int32 intermediates from wrapping in Int16. + for (int row = 0; row < 8; row++) + { + Vector128 packed = Vector128.LoadUnsafe(ref source, (nuint)(row * sourceStride)); + (Vector128 lower, Vector128 upper) = Vector128.Widen(packed); + Vector256 promoted = Finish(Vector256.Create(lower, upper), roundShift, normalizeRectangle); + + promoted.StoreUnsafe(ref promotionBase, (nuint)(row * 8)); + } + + Transpose8x8Int32(ref promotionBase, 8, ref destination, destinationStride, 0, false); + } + + /// + /// Promotes and transposes one 4-by-4 tile of signed sixteen-bit values. + /// + /// The first value of the source tile. + /// The number of signed sixteen-bit values between source rows. + /// The first value of the destination tile. + /// The number of signed thirty-two-bit values between destination rows. + /// The reusable storage for the promoted source tile. + /// The signed AV1 pipeline shift applied after promotion. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose4x4Int16ToInt32( + ref short source, + int sourceStride, + ref int destination, + int destinationStride, + Span promotionBuffer, + int roundShift, + bool normalizeRectangle) + { + ref int promotionBase = ref MemoryMarshal.GetReference(promotionBuffer); + + // The portable vector path retains four independent Int32 axes. Only the lower half is populated because a + // four-wide tile must not read the padded values belonging to its neighboring transform tile. + for (int row = 0; row < 4; row++) + { + Vector128 packed = Load4Short(ref Unsafe.Add(ref source, row * sourceStride)); + (Vector128 promoted, _) = Vector128.Widen(packed); + promoted = Finish(promoted, roundShift, normalizeRectangle); + + promoted.StoreUnsafe(ref promotionBase, (nuint)(row * 4)); + } + + Transpose4x4Int32(ref promotionBase, 4, ref destination, destinationStride, 0, false); + } + /// /// Reverses four signed thirty-two-bit lanes. /// @@ -106,6 +466,15 @@ internal static class Av1Transform2dOperations public static Vector128 Reverse(Vector128 value) => Vector128.ShuffleNative(value, Vector128.Create(3, 2, 1, 0)); + /// + /// Reverses eight signed sixteen-bit lanes. + /// + /// The values to reverse. + /// The values in reverse lane order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Reverse(Vector128 value) + => Vector128.ShuffleNative(value, Vector128.Create((short)7, 6, 5, 4, 3, 2, 1, 0)); + /// /// Reverses eight signed thirty-two-bit lanes. /// @@ -115,6 +484,15 @@ internal static class Av1Transform2dOperations public static Vector256 Reverse(Vector256 value) => Vector256.ShuffleNative(value, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0)); + /// + /// Reverses sixteen signed sixteen-bit lanes. + /// + /// The values to reverse. + /// The values in reverse lane order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Reverse(Vector256 value) + => Vector256.ShuffleNative(value, Vector256.Create((short)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); + /// /// Reverses sixteen signed thirty-two-bit lanes. /// @@ -124,6 +502,17 @@ internal static class Av1Transform2dOperations public static Vector512 Reverse(Vector512 value) => Vector512.ShuffleNative(value, Vector512.Create(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); + /// + /// Reverses thirty-two signed sixteen-bit lanes. + /// + /// The values to reverse. + /// The values in reverse lane order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Reverse(Vector512 value) + => Vector512.ShuffleNative( + value, + Vector512.Create((short)31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); + /// /// Transposes a four-by-four matrix of signed thirty-two-bit lanes. /// @@ -145,6 +534,35 @@ internal static class Av1Transform2dOperations row3 = Vector128_.UnpackHigh(pairs01High.AsInt64(), pairs23High.AsInt64()).AsInt32(); } + /// + /// Transposes one 4-by-4 tile of signed thirty-two-bit values and applies the configured pipeline operations. + /// + /// The first value of the source tile. + /// The number of values between source rows. + /// The first value of the destination tile. + /// The number of values between destination rows. + /// The signed AV1 pipeline shift applied before transposition. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose4x4Int32( + ref int source, + int sourceStride, + ref int destination, + int destinationStride, + int roundShift, + bool normalizeRectangle) + { + Vector128 row0 = Finish(Vector128.LoadUnsafe(ref source), roundShift, normalizeRectangle); + Vector128 row1 = Finish(Vector128.LoadUnsafe(ref source, (nuint)sourceStride), roundShift, normalizeRectangle); + Vector128 row2 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(2 * sourceStride)), roundShift, normalizeRectangle); + Vector128 row3 = Finish(Vector128.LoadUnsafe(ref source, (nuint)(3 * sourceStride)), roundShift, normalizeRectangle); + + Transpose(ref row0, ref row1, ref row2, ref row3); + row0.StoreUnsafe(ref destination); + row1.StoreUnsafe(ref destination, (nuint)destinationStride); + row2.StoreUnsafe(ref destination, (nuint)(2 * destinationStride)); + row3.StoreUnsafe(ref destination, (nuint)(3 * destinationStride)); + } + /// /// Transposes an eight-by-eight matrix of signed thirty-two-bit lanes. /// @@ -201,6 +619,128 @@ internal static class Av1Transform2dOperations row7 = Vector256.Create(column7Lower, column7Upper); } + /// + /// Transposes one 8-by-8 tile of signed thirty-two-bit values and applies the configured pipeline operations. + /// + /// The first value of the source tile. + /// The number of values between source rows. + /// The first value of the destination tile. + /// The number of values between destination rows. + /// The signed AV1 pipeline shift applied before transposition. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose8x8Int32( + ref int source, + int sourceStride, + ref int destination, + int destinationStride, + int roundShift, + bool normalizeRectangle) + { + Vector256 row0 = Finish(Vector256.LoadUnsafe(ref source), roundShift, normalizeRectangle); + Vector256 row1 = Finish(Vector256.LoadUnsafe(ref source, (nuint)sourceStride), roundShift, normalizeRectangle); + Vector256 row2 = Finish(Vector256.LoadUnsafe(ref source, (nuint)(2 * sourceStride)), roundShift, normalizeRectangle); + Vector256 row3 = Finish(Vector256.LoadUnsafe(ref source, (nuint)(3 * sourceStride)), roundShift, normalizeRectangle); + Vector256 row4 = Finish(Vector256.LoadUnsafe(ref source, (nuint)(4 * sourceStride)), roundShift, normalizeRectangle); + Vector256 row5 = Finish(Vector256.LoadUnsafe(ref source, (nuint)(5 * sourceStride)), roundShift, normalizeRectangle); + Vector256 row6 = Finish(Vector256.LoadUnsafe(ref source, (nuint)(6 * sourceStride)), roundShift, normalizeRectangle); + Vector256 row7 = Finish(Vector256.LoadUnsafe(ref source, (nuint)(7 * sourceStride)), roundShift, normalizeRectangle); + + Transpose(ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7); + row0.StoreUnsafe(ref destination); + row1.StoreUnsafe(ref destination, (nuint)destinationStride); + row2.StoreUnsafe(ref destination, (nuint)(2 * destinationStride)); + row3.StoreUnsafe(ref destination, (nuint)(3 * destinationStride)); + row4.StoreUnsafe(ref destination, (nuint)(4 * destinationStride)); + row5.StoreUnsafe(ref destination, (nuint)(5 * destinationStride)); + row6.StoreUnsafe(ref destination, (nuint)(6 * destinationStride)); + row7.StoreUnsafe(ref destination, (nuint)(7 * destinationStride)); + } + + /// + /// Transposes a sixteen-by-sixteen matrix of signed thirty-two-bit values with the libaom AVX-512 staging layout. + /// + /// The first value in the source matrix. + /// The number of values between source rows. + /// The first value in the destination matrix. + /// The number of values between destination rows. + /// The caller-owned storage for sixteen vectors of signed sixty-four-bit lanes. + /// The right shift applied with AV1 signed rounding before transposition. + /// Whether to apply the AV1 square-root-of-two rectangular normalization. + public static void Transpose16x16Avx512( + ref int source, + int sourceStride, + ref int destination, + int destinationStride, + Span scratch, + int roundShift, + bool normalizeRectangle) + { + ref long scratchBase = ref MemoryMarshal.GetReference(scratch); + + // libaom widens the lane grouping after each local interleave rather than retaining all sixteen rows in + // registers. The bounded scratch keeps the live register set small and prevents the JIT from spilling a + // four-stage, sixteen-register cross-vector permutation network into its own stack frame. + for (int row = 0; row < 16; row += 2) + { + Vector512 even = Vector512.LoadUnsafe(ref source, (nuint)(row * sourceStride)); + Vector512 odd = Vector512.LoadUnsafe(ref source, (nuint)((row + 1) * sourceStride)); + even = RoundShift(even, roundShift); + odd = RoundShift(odd, roundShift); + + if (normalizeRectangle) + { + even = Av1Transform1dMath.MultiplyRound(even, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + odd = Av1Transform1dMath.MultiplyRound(odd, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + Avx512F.UnpackLow(even, odd).AsInt64().StoreUnsafe(ref scratchBase, (nuint)(row * 8)); + Avx512F.UnpackHigh(even, odd).AsInt64().StoreUnsafe(ref scratchBase, (nuint)((row + 1) * 8)); + } + + // The second stage exchanges the next row and column bits with 64-bit unpack operations. Each iteration + // reads its complete four-row group before replacing that group in scratch. + for (int row = 0; row < 16; row += 4) + { + for (int offset = 0; offset < 2; offset++) + { + Vector512 lower = Vector512.LoadUnsafe(ref scratchBase, (nuint)((row + offset) * 8)); + Vector512 upper = Vector512.LoadUnsafe(ref scratchBase, (nuint)((row + offset + 2) * 8)); + Avx512F.UnpackLow(lower, upper).StoreUnsafe(ref scratchBase, (nuint)((row + offset) * 8)); + Avx512F.UnpackHigh(lower, upper).StoreUnsafe(ref scratchBase, (nuint)((row + offset + 2) * 8)); + } + } + + Vector512 evenBlockIndices = Vector512.Create(0L, 1L, 8L, 9L, 4L, 5L, 12L, 13L); + Vector512 oddBlockIndices = Vector512.Create(2L, 3L, 10L, 11L, 6L, 7L, 14L, 15L); + + // Highway's LocalInterleaveEvenBlocks and LocalInterleaveOddBlocks exchange the third matrix-index bit with + // one two-table lookup per result. The index vectors address the lower source as 0-7 and the upper as 8-15. + for (int row = 0; row < 16; row += 8) + { + for (int offset = 0; offset < 4; offset++) + { + Vector512 lower = Vector512.LoadUnsafe(ref scratchBase, (nuint)((row + offset) * 8)); + Vector512 upper = Vector512.LoadUnsafe(ref scratchBase, (nuint)((row + offset + 4) * 8)); + Avx512F.PermuteVar8x64x2(lower, evenBlockIndices, upper).StoreUnsafe(ref scratchBase, (nuint)((row + offset) * 8)); + Avx512F.PermuteVar8x64x2(lower, oddBlockIndices, upper).StoreUnsafe(ref scratchBase, (nuint)((row + offset + 4) * 8)); + } + } + + // The final 128-bit-block concatenations complete the transpose. The store order is the fixed permutation + // produced by libaom's three preceding local-interleave stages. + for (int row = 0; row < 8; row++) + { + Vector512 lower = Vector512.LoadUnsafe(ref scratchBase, (nuint)(row * 8)); + Vector512 upper = Vector512.LoadUnsafe(ref scratchBase, (nuint)((row + 8) * 8)); + Vector512 lowerResult = Avx512F.Shuffle4x128(lower.AsInt32(), upper.AsInt32(), 0x44); + Vector512 upperResult = Avx512F.Shuffle4x128(lower.AsInt32(), upper.AsInt32(), 0xEE); + int lowerDestinationRow = Vector512TransposeStoreOrder[row]; + int upperDestinationRow = Vector512TransposeStoreOrder[row + 8]; + lowerResult.StoreUnsafe(ref destination, (nuint)(lowerDestinationRow * destinationStride)); + upperResult.StoreUnsafe(ref destination, (nuint)(upperDestinationRow * destinationStride)); + } + } + /// /// Transposes a sixteen-by-sixteen matrix of signed thirty-two-bit lanes. /// @@ -239,6 +779,153 @@ internal static class Av1Transform2dOperations ref Vector512 row14, ref Vector512 row15) { + if (Avx512F.IsSupported) + { + // Each permutation stage exchanges one row-index bit with the matching column-index bit. After four + // stages the vector index identifies the source column and the lane index identifies the source row. + // This keeps the complete transpose in 512-bit registers instead of decomposing it into 128-bit tiles. + Vector512 stage0Lower = Vector512.Create(0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + Vector512 stage0Upper = Vector512.Create(1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + Vector512 lowerSource = row0; + Vector512 upperSource = row1; + row0 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row1 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row2; + upperSource = row3; + row2 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row3 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row4; + upperSource = row5; + row4 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row5 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row6; + upperSource = row7; + row6 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row7 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row8; + upperSource = row9; + row8 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row9 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row10; + upperSource = row11; + row10 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row11 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row12; + upperSource = row13; + row12 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row13 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + lowerSource = row14; + upperSource = row15; + row14 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Lower, upperSource); + row15 = Avx512F.PermuteVar16x32x2(lowerSource, stage0Upper, upperSource); + + Vector512 stage1Lower = Vector512.Create(0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29); + Vector512 stage1Upper = Vector512.Create(2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31); + lowerSource = row0; + upperSource = row2; + row0 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row2 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row1; + upperSource = row3; + row1 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row3 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row4; + upperSource = row6; + row4 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row6 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row5; + upperSource = row7; + row5 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row7 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row8; + upperSource = row10; + row8 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row10 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row9; + upperSource = row11; + row9 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row11 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row12; + upperSource = row14; + row12 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row14 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + lowerSource = row13; + upperSource = row15; + row13 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Lower, upperSource); + row15 = Avx512F.PermuteVar16x32x2(lowerSource, stage1Upper, upperSource); + + Vector512 stage2Lower = Vector512.Create(0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27); + Vector512 stage2Upper = Vector512.Create(4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31); + lowerSource = row0; + upperSource = row4; + row0 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row4 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row1; + upperSource = row5; + row1 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row5 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row2; + upperSource = row6; + row2 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row6 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row3; + upperSource = row7; + row3 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row7 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row8; + upperSource = row12; + row8 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row12 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row9; + upperSource = row13; + row9 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row13 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row10; + upperSource = row14; + row10 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row14 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + lowerSource = row11; + upperSource = row15; + row11 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Lower, upperSource); + row15 = Avx512F.PermuteVar16x32x2(lowerSource, stage2Upper, upperSource); + + Vector512 stage3Lower = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23); + Vector512 stage3Upper = Vector512.Create(8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31); + lowerSource = row0; + upperSource = row8; + row0 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row8 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row1; + upperSource = row9; + row1 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row9 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row2; + upperSource = row10; + row2 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row10 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row3; + upperSource = row11; + row3 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row11 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row4; + upperSource = row12; + row4 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row12 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row5; + upperSource = row13; + row5 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row13 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row6; + upperSource = row14; + row6 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row14 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + lowerSource = row7; + upperSource = row15; + row7 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Lower, upperSource); + row15 = Avx512F.PermuteVar16x32x2(lowerSource, stage3Upper, upperSource); + return; + } + // A 16x16 transpose consists of four independent 8x8 quadrants. Reusing the established 256-bit transpose // keeps the portable layout path branch-free while the transform arithmetic itself remains in 512-bit lanes. // Preserve the bottom-left quadrant before row8-row15 become upper-column output storage. Emitting those upper @@ -301,4 +988,65 @@ internal static class Av1Transform2dOperations row6 = Vector512.Create(lowerTop6, lowerBottom6); row7 = Vector512.Create(lowerTop7, lowerBottom7); } + + /// + /// Applies the terminal operations which libaom performs before transposing a signed sixteen-bit tile. + /// + /// The packed transform values. + /// The signed AV1 pipeline shift. + /// Whether to apply square-root-of-two rectangular normalization. + /// The shifted and normalized values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Finish(Vector128 value, int roundShift, bool normalizeRectangle) + { + value = RoundShift(value, roundShift); + return normalizeRectangle + ? Forward.Av1ForwardTransformArithmetic>.MultiplyRound( + value, + Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits) + : value; + } + + /// + /// Applies the terminal operations which libaom performs before transposing four signed thirty-two-bit lanes. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Finish(Vector128 value, int roundShift, bool normalizeRectangle) + { + value = RoundShift(value, roundShift); + return normalizeRectangle + ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) + : value; + } + + /// + /// Applies the terminal operations which libaom performs before transposing eight signed thirty-two-bit lanes. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Finish(Vector256 value, int roundShift, bool normalizeRectangle) + { + value = RoundShift(value, roundShift); + return normalizeRectangle + ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) + : value; + } + + /// + /// Loads four signed sixteen-bit values without reading outside the source tile. + /// + /// The first source value. + /// The four source values in the lower vector lanes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Load4Short(ref short source) + => Vector128.Create(Unsafe.As(ref source), 0UL).AsInt16(); + + /// + /// Stores the lower four signed sixteen-bit lanes without writing outside the destination tile. + /// + /// The packed lower lanes. + /// The first destination value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Store4Int16(ulong value, ref short destination) + => Unsafe.As(ref destination) = value; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformVector.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformVector.cs index 70b3eb3a7..75435474a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformVector.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformVector.cs @@ -1,7 +1,9 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -9,12 +11,90 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// Stores the fixed set of SIMD values used by one bulk AV1 transform axis. /// /// The SIMD vector type used for parallel transform lanes. -[InlineArray(Av1Constants.MaxTransformSize)] +[StructLayout(LayoutKind.Sequential)] internal struct Av1TransformVector where TVector : struct { + // Explicit fields give the JIT constant offsets inside large transform operators. The previous inline-array + // helpers were not inlined once those operators exceeded the JIT's expansion budget, causing a call per access. + public TVector V0; + public TVector V1; + public TVector V2; + public TVector V3; + public TVector V4; + public TVector V5; + public TVector V6; + public TVector V7; + public TVector V8; + public TVector V9; + public TVector V10; + public TVector V11; + public TVector V12; + public TVector V13; + public TVector V14; + public TVector V15; + public TVector V16; + public TVector V17; + public TVector V18; + public TVector V19; + public TVector V20; + public TVector V21; + public TVector V22; + public TVector V23; + public TVector V24; + public TVector V25; + public TVector V26; + public TVector V27; + public TVector V28; + public TVector V29; + public TVector V30; + public TVector V31; + public TVector V32; + public TVector V33; + public TVector V34; + public TVector V35; + public TVector V36; + public TVector V37; + public TVector V38; + public TVector V39; + public TVector V40; + public TVector V41; + public TVector V42; + public TVector V43; + public TVector V44; + public TVector V45; + public TVector V46; + public TVector V47; + public TVector V48; + public TVector V49; + public TVector V50; + public TVector V51; + public TVector V52; + public TVector V53; + public TVector V54; + public TVector V55; + public TVector V56; + public TVector V57; + public TVector V58; + public TVector V59; + public TVector V60; + public TVector V61; + public TVector V62; + public TVector V63; + /// - /// The first value in the fixed transform vector. + /// Gets a reference to the SIMD value at the requested transform position. /// - private TVector element0; + /// The zero-based transform position. + /// The SIMD value at the requested position. + [UnscopedRef] + public ref TVector this[int index] + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + ref TVector first = ref Unsafe.As, TVector>(ref this); + return ref Unsafe.Add(ref first, (uint)index); + } + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs index c497ff529..bc5d3ff3f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// @@ -13,6 +15,11 @@ internal static class Av1TransformWorkspace /// public const int Vector512StorageLength = 3 * Av1Constants.MaxTransformSize * 16; + /// + /// The number of integer elements occupied by the libaom-shaped AVX-512 transpose scratch. + /// + public const int Vector512TransposeStorageLength = 16 * 8 * 2; + /// /// The number of integer elements occupied by the three 256-bit transform vectors. /// @@ -26,7 +33,10 @@ internal static class Av1TransformWorkspace /// /// The number of integers required for the largest supported transform block. /// - public const int MaximumLength = (Av1Constants.MaxTransformSize * Av1Constants.MaxTransformSize) + Vector512StorageLength; + public const int MaximumLength = + (2 * Av1Constants.MaxTransformSize * Av1Constants.MaxTransformSize) + + Vector512StorageLength + + Vector512TransposeStorageLength; /// /// Gets the number of integers required for a transform size. @@ -35,6 +45,13 @@ internal static class Av1TransformWorkspace /// The required workspace length. public static int GetRequiredLength(Av1TransformSize transformSize) { - return (transformSize.GetWidth() * transformSize.GetHeight()) + Vector512StorageLength; + // The widest packed transform evaluates thirty-two independent axes together. Small AV1 blocks are padded + // to that lane count, so their workspace requirement is determined by the vector tile rather than the + // coded coefficient count. + int width = Math.Max(transformSize.GetWidth(), Vector512.Count); + int height = Math.Max(transformSize.GetHeight(), Vector512.Count); + int blockLength = width * height; + + return (2 * blockLength) + Vector512StorageLength + Vector512TransposeStorageLength; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs deleted file mode 100644 index 34f211304..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs +++ /dev/null @@ -1,557 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the sixteen-point forward ADST operator. -/// -internal readonly partial struct Av1Adst16Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // The range table is consumed by coefficient-range-checking builds of libaom. ImageSharp preserves the same - // staged arithmetic, while its production path relies on the bit-depth and shift invariants established by - // the two-dimensional transform configuration. - _ = stageRange; - - // Reordering and alternating signs express the ADST as progressively wider symmetric butterflies. - output[0] = input[0]; - output[1] = -input[15]; - output[2] = -input[7]; - output[3] = input[8]; - output[4] = -input[3]; - output[5] = input[12]; - output[6] = input[4]; - output[7] = -input[11]; - output[8] = -input[1]; - output[9] = input[14]; - output[10] = input[6]; - output[11] = -input[9]; - output[12] = input[2]; - output[13] = -input[13]; - output[14] = -input[5]; - output[15] = input[10]; - - // Rotate four independent pairs by pi/4 so the following butterflies can double their span. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Combine adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - output[8] = step[8] + step[10]; - output[9] = step[9] + step[11]; - output[10] = step[8] - step[10]; - output[11] = step[9] - step[11]; - output[12] = step[12] + step[14]; - output[13] = step[13] + step[15]; - output[14] = step[12] - step[14]; - output[15] = step[13] - step[15]; - - // Rotate the upper half of each eight-sample group by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Merge the four-sample groups into two eight-sample butterflies. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - output[8] = step[8] + step[12]; - output[9] = step[9] + step[13]; - output[10] = step[10] + step[14]; - output[11] = step[11] + step[15]; - output[12] = step[8] - step[12]; - output[13] = step[9] - step[13]; - output[14] = step[10] - step[14]; - output[15] = step[11] - step[15]; - - // Rotate the upper eight coefficients with the pi/16 odd-angle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Merge both eight-sample halves into the complete sixteen-sample butterfly. - output[0] = step[0] + step[8]; - output[1] = step[1] + step[9]; - output[2] = step[2] + step[10]; - output[3] = step[3] + step[11]; - output[4] = step[4] + step[12]; - output[5] = step[5] + step[13]; - output[6] = step[6] + step[14]; - output[7] = step[7] + step[15]; - output[8] = step[0] - step[8]; - output[9] = step[1] - step[9]; - output[10] = step[2] - step[10]; - output[11] = step[3] - step[11]; - output[12] = step[4] - step[12]; - output[13] = step[5] - step[13]; - output[14] = step[6] - step[14]; - output[15] = step[7] - step[15]; - - // Apply the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Permute the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[14]; - output[2] = step[3]; - output[3] = step[12]; - output[4] = step[5]; - output[5] = step[10]; - output[6] = step[7]; - output[7] = step[8]; - output[8] = step[9]; - output[9] = step[6]; - output[10] = step[11]; - output[11] = step[4]; - output[12] = step[13]; - output[13] = step[2]; - output[14] = step[15]; - output[15] = step[0]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // The range table is consumed by coefficient-range-checking builds of libaom. ImageSharp preserves the same - // staged arithmetic, while its production path relies on the bit-depth and shift invariants established by - // the two-dimensional transform configuration. - _ = stageRange; - - // Reordering and alternating signs express the ADST as progressively wider symmetric butterflies. - output[0] = input[0]; - output[1] = -input[15]; - output[2] = -input[7]; - output[3] = input[8]; - output[4] = -input[3]; - output[5] = input[12]; - output[6] = input[4]; - output[7] = -input[11]; - output[8] = -input[1]; - output[9] = input[14]; - output[10] = input[6]; - output[11] = -input[9]; - output[12] = input[2]; - output[13] = -input[13]; - output[14] = -input[5]; - output[15] = input[10]; - - // Rotate four independent pairs by pi/4 so the following butterflies can double their span. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Combine adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - output[8] = step[8] + step[10]; - output[9] = step[9] + step[11]; - output[10] = step[8] - step[10]; - output[11] = step[9] - step[11]; - output[12] = step[12] + step[14]; - output[13] = step[13] + step[15]; - output[14] = step[12] - step[14]; - output[15] = step[13] - step[15]; - - // Rotate the upper half of each eight-sample group by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Merge the four-sample groups into two eight-sample butterflies. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - output[8] = step[8] + step[12]; - output[9] = step[9] + step[13]; - output[10] = step[10] + step[14]; - output[11] = step[11] + step[15]; - output[12] = step[8] - step[12]; - output[13] = step[9] - step[13]; - output[14] = step[10] - step[14]; - output[15] = step[11] - step[15]; - - // Rotate the upper eight coefficients with the pi/16 odd-angle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Merge both eight-sample halves into the complete sixteen-sample butterfly. - output[0] = step[0] + step[8]; - output[1] = step[1] + step[9]; - output[2] = step[2] + step[10]; - output[3] = step[3] + step[11]; - output[4] = step[4] + step[12]; - output[5] = step[5] + step[13]; - output[6] = step[6] + step[14]; - output[7] = step[7] + step[15]; - output[8] = step[0] - step[8]; - output[9] = step[1] - step[9]; - output[10] = step[2] - step[10]; - output[11] = step[3] - step[11]; - output[12] = step[4] - step[12]; - output[13] = step[5] - step[13]; - output[14] = step[6] - step[14]; - output[15] = step[7] - step[15]; - - // Apply the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Permute the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[14]; - output[2] = step[3]; - output[3] = step[12]; - output[4] = step[5]; - output[5] = step[10]; - output[6] = step[7]; - output[7] = step[8]; - output[8] = step[9]; - output[9] = step[6]; - output[10] = step[11]; - output[11] = step[4]; - output[12] = step[13]; - output[13] = step[2]; - output[14] = step[15]; - output[15] = step[0]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // The range table is consumed by coefficient-range-checking builds of libaom. ImageSharp preserves the same - // staged arithmetic, while its production path relies on the bit-depth and shift invariants established by - // the two-dimensional transform configuration. - _ = stageRange; - - // Reordering and alternating signs express the ADST as progressively wider symmetric butterflies. - output[0] = input[0]; - output[1] = -input[15]; - output[2] = -input[7]; - output[3] = input[8]; - output[4] = -input[3]; - output[5] = input[12]; - output[6] = input[4]; - output[7] = -input[11]; - output[8] = -input[1]; - output[9] = input[14]; - output[10] = input[6]; - output[11] = -input[9]; - output[12] = input[2]; - output[13] = -input[13]; - output[14] = -input[5]; - output[15] = input[10]; - - // Rotate four independent pairs by pi/4 so the following butterflies can double their span. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Combine adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - output[8] = step[8] + step[10]; - output[9] = step[9] + step[11]; - output[10] = step[8] - step[10]; - output[11] = step[9] - step[11]; - output[12] = step[12] + step[14]; - output[13] = step[13] + step[15]; - output[14] = step[12] - step[14]; - output[15] = step[13] - step[15]; - - // Rotate the upper half of each eight-sample group by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Merge the four-sample groups into two eight-sample butterflies. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - output[8] = step[8] + step[12]; - output[9] = step[9] + step[13]; - output[10] = step[10] + step[14]; - output[11] = step[11] + step[15]; - output[12] = step[8] - step[12]; - output[13] = step[9] - step[13]; - output[14] = step[10] - step[14]; - output[15] = step[11] - step[15]; - - // Rotate the upper eight coefficients with the pi/16 odd-angle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Merge both eight-sample halves into the complete sixteen-sample butterfly. - output[0] = step[0] + step[8]; - output[1] = step[1] + step[9]; - output[2] = step[2] + step[10]; - output[3] = step[3] + step[11]; - output[4] = step[4] + step[12]; - output[5] = step[5] + step[13]; - output[6] = step[6] + step[14]; - output[7] = step[7] + step[15]; - output[8] = step[0] - step[8]; - output[9] = step[1] - step[9]; - output[10] = step[2] - step[10]; - output[11] = step[3] - step[11]; - output[12] = step[4] - step[12]; - output[13] = step[5] - step[13]; - output[14] = step[6] - step[14]; - output[15] = step[7] - step[15]; - - // Apply the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Permute the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[14]; - output[2] = step[3]; - output[3] = step[12]; - output[4] = step[5]; - output[5] = step[10]; - output[6] = step[7]; - output[7] = step[8]; - output[8] = step[9]; - output[9] = step[6]; - output[10] = step[11]; - output[11] = step[4]; - output[12] = step[13]; - output[13] = step[2]; - output[14] = step[15]; - output[15] = step[0]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs index 0fa3f620b..589f33823 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs @@ -4,186 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 16-point AV1 forward asymmetric discrete sine transform operator. +/// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator. /// -internal readonly partial struct Av1Adst16Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Adst16Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 16-point AV1 forward asymmetric discrete sine transform. - /// - /// The sixteen spatial-domain residual values. - /// The sixteen frequency-domain coefficients. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // The range table is consumed by coefficient-range-checking builds of libaom. ImageSharp preserves the same - // staged arithmetic, while its production path relies on the bit-depth and shift invariants established by - // the two-dimensional transform configuration. - _ = stageRange; - - // Reordering and alternating signs express the ADST as progressively wider symmetric butterflies. - output[0] = input[0]; - output[1] = -input[15]; - output[2] = -input[7]; - output[3] = input[8]; - output[4] = -input[3]; - output[5] = input[12]; - output[6] = input[4]; - output[7] = -input[11]; - output[8] = -input[1]; - output[9] = input[14]; - output[10] = input[6]; - output[11] = -input[9]; - output[12] = input[2]; - output[13] = -input[13]; - output[14] = -input[5]; - output[15] = input[10]; - - // Rotate four independent pairs by pi/4 so the following butterflies can double their span. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Combine adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - output[8] = step[8] + step[10]; - output[9] = step[9] + step[11]; - output[10] = step[8] - step[10]; - output[11] = step[9] - step[11]; - output[12] = step[12] + step[14]; - output[13] = step[13] + step[15]; - output[14] = step[12] - step[14]; - output[15] = step[13] - step[15]; - - // Rotate the upper half of each eight-sample group by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Merge the four-sample groups into two eight-sample butterflies. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - output[8] = step[8] + step[12]; - output[9] = step[9] + step[13]; - output[10] = step[10] + step[14]; - output[11] = step[11] + step[15]; - output[12] = step[8] - step[12]; - output[13] = step[9] - step[13]; - output[14] = step[10] - step[14]; - output[15] = step[11] - step[15]; - - // Rotate the upper eight coefficients with the pi/16 odd-angle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Merge both eight-sample halves into the complete sixteen-sample butterfly. - output[0] = step[0] + step[8]; - output[1] = step[1] + step[9]; - output[2] = step[2] + step[10]; - output[3] = step[3] + step[11]; - output[4] = step[4] + step[12]; - output[5] = step[5] + step[13]; - output[6] = step[6] + step[14]; - output[7] = step[7] + step[15]; - output[8] = step[0] - step[8]; - output[9] = step[1] - step[9]; - output[10] = step[2] - step[10]; - output[11] = step[3] - step[11]; - output[12] = step[4] - step[12]; - output[13] = step[5] - step[13]; - output[14] = step[6] - step[14]; - output[15] = step[7] - step[15]; - - // Apply the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Permute the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[14]; - output[2] = step[3]; - output[3] = step[12]; - output[4] = step[5]; - output[5] = step[10]; - output[6] = step[7]; - output[7] = step[8]; - output[8] = step[9]; - output[9] = step[6]; - output[10] = step[11]; - output[11] = step[4]; - output[12] = step[13]; - output[13] = step[2]; - output[14] = step[15]; - output[15] = step[0]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst16(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs deleted file mode 100644 index 25420fc2d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the four-point forward ADST operator. -/// -internal readonly partial struct Av1Adst4Forward1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - /// Applies the four-point matrix to four independent axes. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector128 x0 = input[0]; - Vector128 x1 = input[1]; - Vector128 x2 = input[2]; - Vector128 x3 = input[3]; - - // Expanding the factorization into four weighted sums keeps every lane independent while preserving the - // wrapping 32-bit product and sum behavior used by the optimized fixed-point kernels. - output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[2], x1, sinpi[3], x2, sinpi[4], x3, cosBit); - output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, sinpi[3], x1, 0, x2, -sinpi[3], x3, cosBit); - output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[4], x0, -sinpi[1], x1, -sinpi[3], x2, sinpi[2], x3, cosBit); - output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[4] - sinpi[1], x0, -sinpi[1] - sinpi[2], x1, sinpi[3], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the four-point matrix to eight independent axes. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector256 x0 = input[0]; - Vector256 x1 = input[1]; - Vector256 x2 = input[2]; - Vector256 x3 = input[3]; - - output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[2], x1, sinpi[3], x2, sinpi[4], x3, cosBit); - output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, sinpi[3], x1, 0, x2, -sinpi[3], x3, cosBit); - output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[4], x0, -sinpi[1], x1, -sinpi[3], x2, sinpi[2], x3, cosBit); - output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[4] - sinpi[1], x0, -sinpi[1] - sinpi[2], x1, sinpi[3], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the four-point matrix to sixteen independent axes. - /// - /// The source values for sixteen transform axes. - /// The destination values for sixteen transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector512 x0 = input[0]; - Vector512 x1 = input[1]; - Vector512 x2 = input[2]; - Vector512 x3 = input[3]; - - output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[2], x1, sinpi[3], x2, sinpi[4], x3, cosBit); - output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, sinpi[3], x1, 0, x2, -sinpi[3], x3, cosBit); - output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[4], x0, -sinpi[1], x1, -sinpi[3], x2, sinpi[2], x3, cosBit); - output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[4] - sinpi[1], x0, -sinpi[1] - sinpi[2], x1, sinpi[3], x2, sinpi[2] - sinpi[4], x3, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs index 53e1c2842..cf38e8a31 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs @@ -6,62 +6,14 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// /// Defines the four-point AV1 forward asymmetric discrete sine transform operator. /// -internal readonly partial struct Av1Adst4Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Adst4Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative four-point AV1 forward asymmetric discrete sine transform. - /// - /// The four spatial-domain residual values. - /// The four frequency-domain coefficients. - /// Unused stage storage supplied by the common transform-kernel contract. - /// The fixed-point precision of the sine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = stageRange; - - int x0 = input[0]; - int x1 = input[1]; - int x2 = input[2]; - int x3 = input[3]; - - // Avoid the fixed-point multiplies for the common all-zero residual while producing the exact same result. - if ((x0 | x1 | x2 | x3) == 0) - { - output.Clear(); - return; - } - - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - - // These products are the sparse four-point ADST matrix factorization from the AV1 transform definition. - int s0 = sinpi[1] * x0; - int s1 = sinpi[4] * x0; - int s2 = sinpi[2] * x1; - int s3 = sinpi[1] * x1; - int s4 = sinpi[3] * x2; - int s5 = sinpi[4] * x3; - int s6 = sinpi[2] * x3; - int s7 = x0 + x1 - x3; - - x0 = s0 + s2; - x1 = sinpi[3] * s7; - x2 = s1 - s3; - x3 = s4; - - x0 += s5; - x2 += s6; - - s0 = x0 + x3; - s1 = x1; - s2 = x2 - x3; - s3 = x2 - x0 + x3; - - // The one-dimensional ADST carries a square-root-of-two scale represented by the selected sine table. - output[0] = Av1Math.RoundShift(s0, cosBit); - output[1] = Av1Math.RoundShift(s1, cosBit); - output[2] = Av1Math.RoundShift(s2, cosBit); - output[3] = Av1Math.RoundShift(s3, cosBit); - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst4(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs deleted file mode 100644 index 8083f9f6f..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the eight-point forward ADST operator. -/// -internal readonly partial struct Av1Adst8Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 reorders and signs the inputs so the ADST can be expressed as symmetric butterflies. - output[0] = input[0]; - output[1] = -input[7]; - output[2] = -input[3]; - output[3] = input[4]; - output[4] = -input[1]; - output[5] = input[6]; - output[6] = input[2]; - output[7] = -input[5]; - - // Stage 2 rotates the middle pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 3 combines adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - - // Stage 4 rotates the upper half by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 merges both four-sample halves into the complete eight-sample butterfly. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - - // Stage 6 applies the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 7 permutes the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[6]; - output[2] = step[3]; - output[3] = step[4]; - output[4] = step[5]; - output[5] = step[2]; - output[6] = step[7]; - output[7] = step[0]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 reorders and signs the inputs so the ADST can be expressed as symmetric butterflies. - output[0] = input[0]; - output[1] = -input[7]; - output[2] = -input[3]; - output[3] = input[4]; - output[4] = -input[1]; - output[5] = input[6]; - output[6] = input[2]; - output[7] = -input[5]; - - // Stage 2 rotates the middle pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 3 combines adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - - // Stage 4 rotates the upper half by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 merges both four-sample halves into the complete eight-sample butterfly. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - - // Stage 6 applies the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 7 permutes the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[6]; - output[2] = step[3]; - output[3] = step[4]; - output[4] = step[5]; - output[5] = step[2]; - output[6] = step[7]; - output[7] = step[0]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 reorders and signs the inputs so the ADST can be expressed as symmetric butterflies. - output[0] = input[0]; - output[1] = -input[7]; - output[2] = -input[3]; - output[3] = input[4]; - output[4] = -input[1]; - output[5] = input[6]; - output[6] = input[2]; - output[7] = -input[5]; - - // Stage 2 rotates the middle pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 3 combines adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - - // Stage 4 rotates the upper half by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 merges both four-sample halves into the complete eight-sample butterfly. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - - // Stage 6 applies the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 7 permutes the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[6]; - output[2] = step[3]; - output[3] = step[4]; - output[4] = step[5]; - output[5] = step[2]; - output[6] = step[7]; - output[7] = step[0]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs index a4895006a..e89715f4d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs @@ -6,91 +6,14 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// /// Defines the eight-point AV1 forward asymmetric discrete sine transform operator. /// -internal readonly partial struct Av1Adst8Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Adst8Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative eight-point AV1 forward asymmetric discrete sine transform. - /// - /// The eight spatial-domain residual values. - /// The eight frequency-domain coefficients. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 reorders and signs the inputs so the ADST can be expressed as symmetric butterflies. - output[0] = input[0]; - output[1] = -input[7]; - output[2] = -input[3]; - output[3] = input[4]; - output[4] = -input[1]; - output[5] = input[6]; - output[6] = input[2]; - output[7] = -input[5]; - - // Stage 2 rotates the middle pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 3 combines adjacent rotated pairs into four-sample butterflies. - output[0] = step[0] + step[2]; - output[1] = step[1] + step[3]; - output[2] = step[0] - step[2]; - output[3] = step[1] - step[3]; - output[4] = step[4] + step[6]; - output[5] = step[5] + step[7]; - output[6] = step[4] - step[6]; - output[7] = step[5] - step[7]; - - // Stage 4 rotates the upper half by pi/8 and 3pi/8. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 merges both four-sample halves into the complete eight-sample butterfly. - output[0] = step[0] + step[4]; - output[1] = step[1] + step[5]; - output[2] = step[2] + step[6]; - output[3] = step[3] + step[7]; - output[4] = step[0] - step[4]; - output[5] = step[1] - step[5]; - output[6] = step[2] - step[6]; - output[7] = step[3] - step[7]; - - // Stage 6 applies the terminal odd-frequency rotations that define the ADST basis vectors. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 7 permutes the rotated values into AV1 coefficient order. - output[0] = step[1]; - output[1] = step[6]; - output[2] = step[3]; - output[3] = step[4]; - output[4] = step[5]; - output[5] = step[2]; - output[6] = step[7]; - output[7] = step[0]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst8(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs deleted file mode 100644 index e27bbef0d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the sixteen-point forward DCT operator. -/// -internal readonly partial struct Av1Dct16Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[15]; - output[1] = input[1] + input[14]; - output[2] = input[2] + input[13]; - output[3] = input[3] + input[12]; - output[4] = input[4] + input[11]; - output[5] = input[5] + input[10]; - output[6] = input[6] + input[9]; - output[7] = input[7] + input[8]; - output[8] = -input[8] + input[7]; - output[9] = -input[9] + input[6]; - output[10] = -input[10] + input[5]; - output[11] = -input[11] + input[4]; - output[12] = -input[12] + input[3]; - output[13] = -input[13] + input[2]; - output[14] = -input[14] + input[1]; - output[15] = -input[15] + input[0]; - - // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - - // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - - // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - - // Stage 6 applies the final pi/32 odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - - // Stage 7 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[8]; - output[2] = step[4]; - output[3] = step[12]; - output[4] = step[2]; - output[5] = step[10]; - output[6] = step[6]; - output[7] = step[14]; - output[8] = step[1]; - output[9] = step[9]; - output[10] = step[5]; - output[11] = step[13]; - output[12] = step[3]; - output[13] = step[11]; - output[14] = step[7]; - output[15] = step[15]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[15]; - output[1] = input[1] + input[14]; - output[2] = input[2] + input[13]; - output[3] = input[3] + input[12]; - output[4] = input[4] + input[11]; - output[5] = input[5] + input[10]; - output[6] = input[6] + input[9]; - output[7] = input[7] + input[8]; - output[8] = -input[8] + input[7]; - output[9] = -input[9] + input[6]; - output[10] = -input[10] + input[5]; - output[11] = -input[11] + input[4]; - output[12] = -input[12] + input[3]; - output[13] = -input[13] + input[2]; - output[14] = -input[14] + input[1]; - output[15] = -input[15] + input[0]; - - // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - - // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - - // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - - // Stage 6 applies the final pi/32 odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - - // Stage 7 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[8]; - output[2] = step[4]; - output[3] = step[12]; - output[4] = step[2]; - output[5] = step[10]; - output[6] = step[6]; - output[7] = step[14]; - output[8] = step[1]; - output[9] = step[9]; - output[10] = step[5]; - output[11] = step[13]; - output[12] = step[3]; - output[13] = step[11]; - output[14] = step[7]; - output[15] = step[15]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[15]; - output[1] = input[1] + input[14]; - output[2] = input[2] + input[13]; - output[3] = input[3] + input[12]; - output[4] = input[4] + input[11]; - output[5] = input[5] + input[10]; - output[6] = input[6] + input[9]; - output[7] = input[7] + input[8]; - output[8] = -input[8] + input[7]; - output[9] = -input[9] + input[6]; - output[10] = -input[10] + input[5]; - output[11] = -input[11] + input[4]; - output[12] = -input[12] + input[3]; - output[13] = -input[13] + input[2]; - output[14] = -input[14] + input[1]; - output[15] = -input[15] + input[0]; - - // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - - // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - - // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - - // Stage 6 applies the final pi/32 odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - - // Stage 7 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[8]; - output[2] = step[4]; - output[3] = step[12]; - output[4] = step[2]; - output[5] = step[10]; - output[6] = step[6]; - output[7] = step[14]; - output[8] = step[1]; - output[9] = step[9]; - output[10] = step[5]; - output[11] = step[13]; - output[12] = step[3]; - output[13] = step[11]; - output[14] = step[7]; - output[15] = step[15]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs index cd5a6b4a4..e4f768a7f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs @@ -4,149 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 16-point AV1 forward discrete cosine transform operator. +/// Defines the sixteen-point AV1 forward discrete cosine transform operator. /// -internal readonly partial struct Av1Dct16Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Dct16Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 16-point AV1 forward discrete cosine transform. - /// - /// The sixteen spatial-domain residual values. - /// The sixteen frequency-domain coefficients. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[15]; - output[1] = input[1] + input[14]; - output[2] = input[2] + input[13]; - output[3] = input[3] + input[12]; - output[4] = input[4] + input[11]; - output[5] = input[5] + input[10]; - output[6] = input[6] + input[9]; - output[7] = input[7] + input[8]; - output[8] = -input[8] + input[7]; - output[9] = -input[9] + input[6]; - output[10] = -input[10] + input[5]; - output[11] = -input[11] + input[4]; - output[12] = -input[12] + input[3]; - output[13] = -input[13] + input[2]; - output[14] = -input[14] + input[1]; - output[15] = -input[15] + input[0]; - - // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - - // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - - // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - - // Stage 6 applies the final pi/32 odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - - // Stage 7 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[8]; - output[2] = step[4]; - output[3] = step[12]; - output[4] = step[2]; - output[5] = step[10]; - output[6] = step[6]; - output[7] = step[14]; - output[8] = step[1]; - output[9] = step[9]; - output[10] = step[5]; - output[11] = step[13]; - output[12] = step[3]; - output[13] = step[11]; - output[14] = step[7]; - output[15] = step[15]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct16(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs deleted file mode 100644 index 9742c53f6..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs +++ /dev/null @@ -1,986 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the thirty-two-point forward DCT operator. -/// -internal readonly partial struct Av1Dct32Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[31]; - output[1] = input[1] + input[30]; - output[2] = input[2] + input[29]; - output[3] = input[3] + input[28]; - output[4] = input[4] + input[27]; - output[5] = input[5] + input[26]; - output[6] = input[6] + input[25]; - output[7] = input[7] + input[24]; - output[8] = input[8] + input[23]; - output[9] = input[9] + input[22]; - output[10] = input[10] + input[21]; - output[11] = input[11] + input[20]; - output[12] = input[12] + input[19]; - output[13] = input[13] + input[18]; - output[14] = input[14] + input[17]; - output[15] = input[15] + input[16]; - output[16] = -input[16] + input[15]; - output[17] = -input[17] + input[14]; - output[18] = -input[18] + input[13]; - output[19] = -input[19] + input[12]; - output[20] = -input[20] + input[11]; - output[21] = -input[21] + input[10]; - output[22] = -input[22] + input[9]; - output[23] = -input[23] + input[8]; - output[24] = -input[24] + input[7]; - output[25] = -input[25] + input[6]; - output[26] = -input[26] + input[5]; - output[27] = -input[27] + input[4]; - output[28] = -input[28] + input[3]; - output[29] = -input[29] + input[2]; - output[30] = -input[30] + input[1]; - output[31] = -input[31] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[15]; - step[1] = output[1] + output[14]; - step[2] = output[2] + output[13]; - step[3] = output[3] + output[12]; - step[4] = output[4] + output[11]; - step[5] = output[5] + output[10]; - step[6] = output[6] + output[9]; - step[7] = output[7] + output[8]; - step[8] = -output[8] + output[7]; - step[9] = -output[9] + output[6]; - step[10] = -output[10] + output[5]; - step[11] = -output[11] + output[4]; - step[12] = -output[12] + output[3]; - step[13] = -output[13] + output[2]; - step[14] = -output[14] + output[1]; - step[15] = -output[15] + output[0]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[24], cospi[32], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[25], cospi[32], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[26], cospi[32], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[27], cospi[32], output[20], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[7]; - output[1] = step[1] + step[6]; - output[2] = step[2] + step[5]; - output[3] = step[3] + step[4]; - output[4] = -step[4] + step[3]; - output[5] = -step[5] + step[2]; - output[6] = -step[6] + step[1]; - output[7] = -step[7] + step[0]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[12], cospi[32], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[13], cospi[32], step[10], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = step[16] + step[23]; - output[17] = step[17] + step[22]; - output[18] = step[18] + step[21]; - output[19] = step[19] + step[20]; - output[20] = -step[20] + step[19]; - output[21] = -step[21] + step[18]; - output[22] = -step[22] + step[17]; - output[23] = -step[23] + step[16]; - output[24] = -step[24] + step[31]; - output[25] = -step[25] + step[30]; - output[26] = -step[26] + step[29]; - output[27] = -step[27] + step[28]; - output[28] = step[28] + step[27]; - output[29] = step[29] + step[26]; - output[30] = step[30] + step[25]; - output[31] = step[31] + step[24]; - - // Stage 4 continues the factorization as independent eight-sample groups. - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - step[8] = output[8] + output[11]; - step[9] = output[9] + output[10]; - step[10] = -output[10] + output[9]; - step[11] = -output[11] + output[8]; - step[12] = -output[12] + output[15]; - step[13] = -output[13] + output[14]; - step[14] = output[14] + output[13]; - step[15] = output[15] + output[12]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(cospi[48], output[26], -cospi[16], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[48], output[27], -cospi[16], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[16], output[28], cospi[48], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[16], output[29], cospi[48], output[18], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(cospi[48], step[13], -cospi[16], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[16], step[14], cospi[48], step[9], cosBit); - output[15] = step[15]; - output[16] = step[16] + step[19]; - output[17] = step[17] + step[18]; - output[18] = -step[18] + step[17]; - output[19] = -step[19] + step[16]; - output[20] = -step[20] + step[23]; - output[21] = -step[21] + step[22]; - output[22] = step[22] + step[21]; - output[23] = step[23] + step[20]; - output[24] = step[24] + step[27]; - output[25] = step[25] + step[26]; - output[26] = -step[26] + step[25]; - output[27] = -step[27] + step[24]; - output[28] = -step[28] + step[31]; - output[29] = -step[29] + step[30]; - output[30] = step[30] + step[29]; - output[31] = step[31] + step[28]; - - // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - step[8] = output[8] + output[9]; - step[9] = -output[9] + output[8]; - step[10] = -output[10] + output[11]; - step[11] = output[11] + output[10]; - step[12] = output[12] + output[13]; - step[13] = -output[13] + output[12]; - step[14] = -output[14] + output[15]; - step[15] = output[15] + output[14]; - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(cospi[24], output[25], -cospi[40], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[40], output[26], cospi[24], output[21], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(cospi[56], output[29], -cospi[8], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[8], output[30], cospi[56], output[17], cosBit); - step[31] = output[31]; - - // Stage 7 applies the pi/32 rotations to the next odd-frequency level. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[12], step[12], -cospi[52], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[44], step[13], -cospi[20], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[28], step[14], -cospi[36], step[9], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[60], step[15], -cospi[4], step[8], cosBit); - output[16] = step[16] + step[17]; - output[17] = -step[17] + step[16]; - output[18] = -step[18] + step[19]; - output[19] = step[19] + step[18]; - output[20] = step[20] + step[21]; - output[21] = -step[21] + step[20]; - output[22] = -step[22] + step[23]; - output[23] = step[23] + step[22]; - output[24] = step[24] + step[25]; - output[25] = -step[25] + step[24]; - output[26] = -step[26] + step[27]; - output[27] = step[27] + step[26]; - output[28] = step[28] + step[29]; - output[29] = -step[29] + step[28]; - output[30] = -step[30] + step[31]; - output[31] = step[31] + step[30]; - - // Stage 8 merges the final odd-frequency pairs before their terminal rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[6], output[24], -cospi[58], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[38], output[25], -cospi[26], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[22], output[26], -cospi[42], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[54], output[27], -cospi[10], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[14], output[28], -cospi[50], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[46], output[29], -cospi[18], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[30], output[30], -cospi[34], output[17], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[62], output[31], -cospi[2], output[16], cosBit); - - // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[16]; - output[2] = step[8]; - output[3] = step[24]; - output[4] = step[4]; - output[5] = step[20]; - output[6] = step[12]; - output[7] = step[28]; - output[8] = step[2]; - output[9] = step[18]; - output[10] = step[10]; - output[11] = step[26]; - output[12] = step[6]; - output[13] = step[22]; - output[14] = step[14]; - output[15] = step[30]; - output[16] = step[1]; - output[17] = step[17]; - output[18] = step[9]; - output[19] = step[25]; - output[20] = step[5]; - output[21] = step[21]; - output[22] = step[13]; - output[23] = step[29]; - output[24] = step[3]; - output[25] = step[19]; - output[26] = step[11]; - output[27] = step[27]; - output[28] = step[7]; - output[29] = step[23]; - output[30] = step[15]; - output[31] = step[31]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[31]; - output[1] = input[1] + input[30]; - output[2] = input[2] + input[29]; - output[3] = input[3] + input[28]; - output[4] = input[4] + input[27]; - output[5] = input[5] + input[26]; - output[6] = input[6] + input[25]; - output[7] = input[7] + input[24]; - output[8] = input[8] + input[23]; - output[9] = input[9] + input[22]; - output[10] = input[10] + input[21]; - output[11] = input[11] + input[20]; - output[12] = input[12] + input[19]; - output[13] = input[13] + input[18]; - output[14] = input[14] + input[17]; - output[15] = input[15] + input[16]; - output[16] = -input[16] + input[15]; - output[17] = -input[17] + input[14]; - output[18] = -input[18] + input[13]; - output[19] = -input[19] + input[12]; - output[20] = -input[20] + input[11]; - output[21] = -input[21] + input[10]; - output[22] = -input[22] + input[9]; - output[23] = -input[23] + input[8]; - output[24] = -input[24] + input[7]; - output[25] = -input[25] + input[6]; - output[26] = -input[26] + input[5]; - output[27] = -input[27] + input[4]; - output[28] = -input[28] + input[3]; - output[29] = -input[29] + input[2]; - output[30] = -input[30] + input[1]; - output[31] = -input[31] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[15]; - step[1] = output[1] + output[14]; - step[2] = output[2] + output[13]; - step[3] = output[3] + output[12]; - step[4] = output[4] + output[11]; - step[5] = output[5] + output[10]; - step[6] = output[6] + output[9]; - step[7] = output[7] + output[8]; - step[8] = -output[8] + output[7]; - step[9] = -output[9] + output[6]; - step[10] = -output[10] + output[5]; - step[11] = -output[11] + output[4]; - step[12] = -output[12] + output[3]; - step[13] = -output[13] + output[2]; - step[14] = -output[14] + output[1]; - step[15] = -output[15] + output[0]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[24], cospi[32], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[25], cospi[32], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[26], cospi[32], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[27], cospi[32], output[20], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[7]; - output[1] = step[1] + step[6]; - output[2] = step[2] + step[5]; - output[3] = step[3] + step[4]; - output[4] = -step[4] + step[3]; - output[5] = -step[5] + step[2]; - output[6] = -step[6] + step[1]; - output[7] = -step[7] + step[0]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[12], cospi[32], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[13], cospi[32], step[10], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = step[16] + step[23]; - output[17] = step[17] + step[22]; - output[18] = step[18] + step[21]; - output[19] = step[19] + step[20]; - output[20] = -step[20] + step[19]; - output[21] = -step[21] + step[18]; - output[22] = -step[22] + step[17]; - output[23] = -step[23] + step[16]; - output[24] = -step[24] + step[31]; - output[25] = -step[25] + step[30]; - output[26] = -step[26] + step[29]; - output[27] = -step[27] + step[28]; - output[28] = step[28] + step[27]; - output[29] = step[29] + step[26]; - output[30] = step[30] + step[25]; - output[31] = step[31] + step[24]; - - // Stage 4 continues the factorization as independent eight-sample groups. - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - step[8] = output[8] + output[11]; - step[9] = output[9] + output[10]; - step[10] = -output[10] + output[9]; - step[11] = -output[11] + output[8]; - step[12] = -output[12] + output[15]; - step[13] = -output[13] + output[14]; - step[14] = output[14] + output[13]; - step[15] = output[15] + output[12]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(cospi[48], output[26], -cospi[16], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[48], output[27], -cospi[16], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[16], output[28], cospi[48], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[16], output[29], cospi[48], output[18], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(cospi[48], step[13], -cospi[16], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[16], step[14], cospi[48], step[9], cosBit); - output[15] = step[15]; - output[16] = step[16] + step[19]; - output[17] = step[17] + step[18]; - output[18] = -step[18] + step[17]; - output[19] = -step[19] + step[16]; - output[20] = -step[20] + step[23]; - output[21] = -step[21] + step[22]; - output[22] = step[22] + step[21]; - output[23] = step[23] + step[20]; - output[24] = step[24] + step[27]; - output[25] = step[25] + step[26]; - output[26] = -step[26] + step[25]; - output[27] = -step[27] + step[24]; - output[28] = -step[28] + step[31]; - output[29] = -step[29] + step[30]; - output[30] = step[30] + step[29]; - output[31] = step[31] + step[28]; - - // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - step[8] = output[8] + output[9]; - step[9] = -output[9] + output[8]; - step[10] = -output[10] + output[11]; - step[11] = output[11] + output[10]; - step[12] = output[12] + output[13]; - step[13] = -output[13] + output[12]; - step[14] = -output[14] + output[15]; - step[15] = output[15] + output[14]; - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(cospi[24], output[25], -cospi[40], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[40], output[26], cospi[24], output[21], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(cospi[56], output[29], -cospi[8], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[8], output[30], cospi[56], output[17], cosBit); - step[31] = output[31]; - - // Stage 7 applies the pi/32 rotations to the next odd-frequency level. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[12], step[12], -cospi[52], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[44], step[13], -cospi[20], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[28], step[14], -cospi[36], step[9], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[60], step[15], -cospi[4], step[8], cosBit); - output[16] = step[16] + step[17]; - output[17] = -step[17] + step[16]; - output[18] = -step[18] + step[19]; - output[19] = step[19] + step[18]; - output[20] = step[20] + step[21]; - output[21] = -step[21] + step[20]; - output[22] = -step[22] + step[23]; - output[23] = step[23] + step[22]; - output[24] = step[24] + step[25]; - output[25] = -step[25] + step[24]; - output[26] = -step[26] + step[27]; - output[27] = step[27] + step[26]; - output[28] = step[28] + step[29]; - output[29] = -step[29] + step[28]; - output[30] = -step[30] + step[31]; - output[31] = step[31] + step[30]; - - // Stage 8 merges the final odd-frequency pairs before their terminal rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[6], output[24], -cospi[58], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[38], output[25], -cospi[26], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[22], output[26], -cospi[42], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[54], output[27], -cospi[10], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[14], output[28], -cospi[50], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[46], output[29], -cospi[18], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[30], output[30], -cospi[34], output[17], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[62], output[31], -cospi[2], output[16], cosBit); - - // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[16]; - output[2] = step[8]; - output[3] = step[24]; - output[4] = step[4]; - output[5] = step[20]; - output[6] = step[12]; - output[7] = step[28]; - output[8] = step[2]; - output[9] = step[18]; - output[10] = step[10]; - output[11] = step[26]; - output[12] = step[6]; - output[13] = step[22]; - output[14] = step[14]; - output[15] = step[30]; - output[16] = step[1]; - output[17] = step[17]; - output[18] = step[9]; - output[19] = step[25]; - output[20] = step[5]; - output[21] = step[21]; - output[22] = step[13]; - output[23] = step[29]; - output[24] = step[3]; - output[25] = step[19]; - output[26] = step[11]; - output[27] = step[27]; - output[28] = step[7]; - output[29] = step[23]; - output[30] = step[15]; - output[31] = step[31]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[31]; - output[1] = input[1] + input[30]; - output[2] = input[2] + input[29]; - output[3] = input[3] + input[28]; - output[4] = input[4] + input[27]; - output[5] = input[5] + input[26]; - output[6] = input[6] + input[25]; - output[7] = input[7] + input[24]; - output[8] = input[8] + input[23]; - output[9] = input[9] + input[22]; - output[10] = input[10] + input[21]; - output[11] = input[11] + input[20]; - output[12] = input[12] + input[19]; - output[13] = input[13] + input[18]; - output[14] = input[14] + input[17]; - output[15] = input[15] + input[16]; - output[16] = -input[16] + input[15]; - output[17] = -input[17] + input[14]; - output[18] = -input[18] + input[13]; - output[19] = -input[19] + input[12]; - output[20] = -input[20] + input[11]; - output[21] = -input[21] + input[10]; - output[22] = -input[22] + input[9]; - output[23] = -input[23] + input[8]; - output[24] = -input[24] + input[7]; - output[25] = -input[25] + input[6]; - output[26] = -input[26] + input[5]; - output[27] = -input[27] + input[4]; - output[28] = -input[28] + input[3]; - output[29] = -input[29] + input[2]; - output[30] = -input[30] + input[1]; - output[31] = -input[31] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[15]; - step[1] = output[1] + output[14]; - step[2] = output[2] + output[13]; - step[3] = output[3] + output[12]; - step[4] = output[4] + output[11]; - step[5] = output[5] + output[10]; - step[6] = output[6] + output[9]; - step[7] = output[7] + output[8]; - step[8] = -output[8] + output[7]; - step[9] = -output[9] + output[6]; - step[10] = -output[10] + output[5]; - step[11] = -output[11] + output[4]; - step[12] = -output[12] + output[3]; - step[13] = -output[13] + output[2]; - step[14] = -output[14] + output[1]; - step[15] = -output[15] + output[0]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[24], cospi[32], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[25], cospi[32], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[26], cospi[32], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[27], cospi[32], output[20], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[7]; - output[1] = step[1] + step[6]; - output[2] = step[2] + step[5]; - output[3] = step[3] + step[4]; - output[4] = -step[4] + step[3]; - output[5] = -step[5] + step[2]; - output[6] = -step[6] + step[1]; - output[7] = -step[7] + step[0]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[12], cospi[32], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[13], cospi[32], step[10], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = step[16] + step[23]; - output[17] = step[17] + step[22]; - output[18] = step[18] + step[21]; - output[19] = step[19] + step[20]; - output[20] = -step[20] + step[19]; - output[21] = -step[21] + step[18]; - output[22] = -step[22] + step[17]; - output[23] = -step[23] + step[16]; - output[24] = -step[24] + step[31]; - output[25] = -step[25] + step[30]; - output[26] = -step[26] + step[29]; - output[27] = -step[27] + step[28]; - output[28] = step[28] + step[27]; - output[29] = step[29] + step[26]; - output[30] = step[30] + step[25]; - output[31] = step[31] + step[24]; - - // Stage 4 continues the factorization as independent eight-sample groups. - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - step[8] = output[8] + output[11]; - step[9] = output[9] + output[10]; - step[10] = -output[10] + output[9]; - step[11] = -output[11] + output[8]; - step[12] = -output[12] + output[15]; - step[13] = -output[13] + output[14]; - step[14] = output[14] + output[13]; - step[15] = output[15] + output[12]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(cospi[48], output[26], -cospi[16], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[48], output[27], -cospi[16], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[16], output[28], cospi[48], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[16], output[29], cospi[48], output[18], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(cospi[48], step[13], -cospi[16], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[16], step[14], cospi[48], step[9], cosBit); - output[15] = step[15]; - output[16] = step[16] + step[19]; - output[17] = step[17] + step[18]; - output[18] = -step[18] + step[17]; - output[19] = -step[19] + step[16]; - output[20] = -step[20] + step[23]; - output[21] = -step[21] + step[22]; - output[22] = step[22] + step[21]; - output[23] = step[23] + step[20]; - output[24] = step[24] + step[27]; - output[25] = step[25] + step[26]; - output[26] = -step[26] + step[25]; - output[27] = -step[27] + step[24]; - output[28] = -step[28] + step[31]; - output[29] = -step[29] + step[30]; - output[30] = step[30] + step[29]; - output[31] = step[31] + step[28]; - - // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - step[8] = output[8] + output[9]; - step[9] = -output[9] + output[8]; - step[10] = -output[10] + output[11]; - step[11] = output[11] + output[10]; - step[12] = output[12] + output[13]; - step[13] = -output[13] + output[12]; - step[14] = -output[14] + output[15]; - step[15] = output[15] + output[14]; - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(cospi[24], output[25], -cospi[40], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[40], output[26], cospi[24], output[21], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(cospi[56], output[29], -cospi[8], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[8], output[30], cospi[56], output[17], cosBit); - step[31] = output[31]; - - // Stage 7 applies the pi/32 rotations to the next odd-frequency level. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[12], step[12], -cospi[52], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[44], step[13], -cospi[20], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[28], step[14], -cospi[36], step[9], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[60], step[15], -cospi[4], step[8], cosBit); - output[16] = step[16] + step[17]; - output[17] = -step[17] + step[16]; - output[18] = -step[18] + step[19]; - output[19] = step[19] + step[18]; - output[20] = step[20] + step[21]; - output[21] = -step[21] + step[20]; - output[22] = -step[22] + step[23]; - output[23] = step[23] + step[22]; - output[24] = step[24] + step[25]; - output[25] = -step[25] + step[24]; - output[26] = -step[26] + step[27]; - output[27] = step[27] + step[26]; - output[28] = step[28] + step[29]; - output[29] = -step[29] + step[28]; - output[30] = -step[30] + step[31]; - output[31] = step[31] + step[30]; - - // Stage 8 merges the final odd-frequency pairs before their terminal rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[6], output[24], -cospi[58], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[38], output[25], -cospi[26], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[22], output[26], -cospi[42], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[54], output[27], -cospi[10], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[14], output[28], -cospi[50], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[46], output[29], -cospi[18], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[30], output[30], -cospi[34], output[17], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[62], output[31], -cospi[2], output[16], cosBit); - - // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[16]; - output[2] = step[8]; - output[3] = step[24]; - output[4] = step[4]; - output[5] = step[20]; - output[6] = step[12]; - output[7] = step[28]; - output[8] = step[2]; - output[9] = step[18]; - output[10] = step[10]; - output[11] = step[26]; - output[12] = step[6]; - output[13] = step[22]; - output[14] = step[14]; - output[15] = step[30]; - output[16] = step[1]; - output[17] = step[17]; - output[18] = step[9]; - output[19] = step[25]; - output[20] = step[5]; - output[21] = step[21]; - output[22] = step[13]; - output[23] = step[29]; - output[24] = step[3]; - output[25] = step[19]; - output[26] = step[11]; - output[27] = step[27]; - output[28] = step[7]; - output[29] = step[23]; - output[30] = step[15]; - output[31] = step[31]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs index 6b528a6f0..4efb9f27e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs @@ -4,329 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 32-point AV1 forward discrete cosine transform operator. +/// Defines the thirty-two-point AV1 forward discrete cosine transform operator. /// -internal readonly partial struct Av1Dct32Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Dct32Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 32-point AV1 forward discrete cosine transform. - /// - /// The 32 spatial-domain residual values. - /// The 32 frequency-domain coefficients. - /// The 32-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[31]; - output[1] = input[1] + input[30]; - output[2] = input[2] + input[29]; - output[3] = input[3] + input[28]; - output[4] = input[4] + input[27]; - output[5] = input[5] + input[26]; - output[6] = input[6] + input[25]; - output[7] = input[7] + input[24]; - output[8] = input[8] + input[23]; - output[9] = input[9] + input[22]; - output[10] = input[10] + input[21]; - output[11] = input[11] + input[20]; - output[12] = input[12] + input[19]; - output[13] = input[13] + input[18]; - output[14] = input[14] + input[17]; - output[15] = input[15] + input[16]; - output[16] = -input[16] + input[15]; - output[17] = -input[17] + input[14]; - output[18] = -input[18] + input[13]; - output[19] = -input[19] + input[12]; - output[20] = -input[20] + input[11]; - output[21] = -input[21] + input[10]; - output[22] = -input[22] + input[9]; - output[23] = -input[23] + input[8]; - output[24] = -input[24] + input[7]; - output[25] = -input[25] + input[6]; - output[26] = -input[26] + input[5]; - output[27] = -input[27] + input[4]; - output[28] = -input[28] + input[3]; - output[29] = -input[29] + input[2]; - output[30] = -input[30] + input[1]; - output[31] = -input[31] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[15]; - step[1] = output[1] + output[14]; - step[2] = output[2] + output[13]; - step[3] = output[3] + output[12]; - step[4] = output[4] + output[11]; - step[5] = output[5] + output[10]; - step[6] = output[6] + output[9]; - step[7] = output[7] + output[8]; - step[8] = -output[8] + output[7]; - step[9] = -output[9] + output[6]; - step[10] = -output[10] + output[5]; - step[11] = -output[11] + output[4]; - step[12] = -output[12] + output[3]; - step[13] = -output[13] + output[2]; - step[14] = -output[14] + output[1]; - step[15] = -output[15] + output[0]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[24], cospi[32], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[25], cospi[32], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[26], cospi[32], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[27], cospi[32], output[20], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[7]; - output[1] = step[1] + step[6]; - output[2] = step[2] + step[5]; - output[3] = step[3] + step[4]; - output[4] = -step[4] + step[3]; - output[5] = -step[5] + step[2]; - output[6] = -step[6] + step[1]; - output[7] = -step[7] + step[0]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[12], cospi[32], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[13], cospi[32], step[10], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = step[16] + step[23]; - output[17] = step[17] + step[22]; - output[18] = step[18] + step[21]; - output[19] = step[19] + step[20]; - output[20] = -step[20] + step[19]; - output[21] = -step[21] + step[18]; - output[22] = -step[22] + step[17]; - output[23] = -step[23] + step[16]; - output[24] = -step[24] + step[31]; - output[25] = -step[25] + step[30]; - output[26] = -step[26] + step[29]; - output[27] = -step[27] + step[28]; - output[28] = step[28] + step[27]; - output[29] = step[29] + step[26]; - output[30] = step[30] + step[25]; - output[31] = step[31] + step[24]; - - // Stage 4 continues the factorization as independent eight-sample groups. - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - step[8] = output[8] + output[11]; - step[9] = output[9] + output[10]; - step[10] = -output[10] + output[9]; - step[11] = -output[11] + output[8]; - step[12] = -output[12] + output[15]; - step[13] = -output[13] + output[14]; - step[14] = output[14] + output[13]; - step[15] = output[15] + output[12]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(cospi[48], output[26], -cospi[16], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[48], output[27], -cospi[16], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[16], output[28], cospi[48], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[16], output[29], cospi[48], output[18], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(cospi[48], step[13], -cospi[16], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[16], step[14], cospi[48], step[9], cosBit); - output[15] = step[15]; - output[16] = step[16] + step[19]; - output[17] = step[17] + step[18]; - output[18] = -step[18] + step[17]; - output[19] = -step[19] + step[16]; - output[20] = -step[20] + step[23]; - output[21] = -step[21] + step[22]; - output[22] = step[22] + step[21]; - output[23] = step[23] + step[20]; - output[24] = step[24] + step[27]; - output[25] = step[25] + step[26]; - output[26] = -step[26] + step[25]; - output[27] = -step[27] + step[24]; - output[28] = -step[28] + step[31]; - output[29] = -step[29] + step[30]; - output[30] = step[30] + step[29]; - output[31] = step[31] + step[28]; - - // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - step[8] = output[8] + output[9]; - step[9] = -output[9] + output[8]; - step[10] = -output[10] + output[11]; - step[11] = output[11] + output[10]; - step[12] = output[12] + output[13]; - step[13] = -output[13] + output[12]; - step[14] = -output[14] + output[15]; - step[15] = output[15] + output[14]; - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(cospi[24], output[25], -cospi[40], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[40], output[26], cospi[24], output[21], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(cospi[56], output[29], -cospi[8], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[8], output[30], cospi[56], output[17], cosBit); - step[31] = output[31]; - - // Stage 7 applies the pi/32 rotations to the next odd-frequency level. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[12], step[12], -cospi[52], step[11], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[44], step[13], -cospi[20], step[10], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[28], step[14], -cospi[36], step[9], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[60], step[15], -cospi[4], step[8], cosBit); - output[16] = step[16] + step[17]; - output[17] = -step[17] + step[16]; - output[18] = -step[18] + step[19]; - output[19] = step[19] + step[18]; - output[20] = step[20] + step[21]; - output[21] = -step[21] + step[20]; - output[22] = -step[22] + step[23]; - output[23] = step[23] + step[22]; - output[24] = step[24] + step[25]; - output[25] = -step[25] + step[24]; - output[26] = -step[26] + step[27]; - output[27] = step[27] + step[26]; - output[28] = step[28] + step[29]; - output[29] = -step[29] + step[28]; - output[30] = -step[30] + step[31]; - output[31] = step[31] + step[30]; - - // Stage 8 merges the final odd-frequency pairs before their terminal rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[6], output[24], -cospi[58], output[23], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[38], output[25], -cospi[26], output[22], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[22], output[26], -cospi[42], output[21], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[54], output[27], -cospi[10], output[20], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[14], output[28], -cospi[50], output[19], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[46], output[29], -cospi[18], output[18], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[30], output[30], -cospi[34], output[17], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[62], output[31], -cospi[2], output[16], cosBit); - - // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[16]; - output[2] = step[8]; - output[3] = step[24]; - output[4] = step[4]; - output[5] = step[20]; - output[6] = step[12]; - output[7] = step[28]; - output[8] = step[2]; - output[9] = step[18]; - output[10] = step[10]; - output[11] = step[26]; - output[12] = step[6]; - output[13] = step[22]; - output[14] = step[14]; - output[15] = step[30]; - output[16] = step[1]; - output[17] = step[17]; - output[18] = step[9]; - output[19] = step[25]; - output[20] = step[5]; - output[21] = step[21]; - output[22] = step[13]; - output[23] = step[29]; - output[24] = step[3]; - output[25] = step[19]; - output[26] = step[11]; - output[27] = step[27]; - output[28] = step[7]; - output[29] = step[23]; - output[30] = step[15]; - output[31] = step[31]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct32(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs deleted file mode 100644 index ef74fc886..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the four-point forward DCT operator. -/// -internal readonly partial struct Av1Dct4Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - _ = stageRange; - - // Mirror butterflies separate the even and odd spatial symmetries used by the four DCT basis vectors. - output[0] = input[0] + input[3]; - output[1] = input[1] + input[2]; - output[2] = input[1] - input[2]; - output[3] = input[0] - input[3]; - - // Each half-butterfly keeps the optimized kernels' wrapping 32-bit arithmetic before applying the - // normative fixed-point rounding shift. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - - // The staged order groups butterfly partners; AV1 coefficient order interleaves their frequency indices. - output[0] = step[0]; - output[1] = step[2]; - output[2] = step[1]; - output[3] = step[3]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - _ = stageRange; - - // Mirror butterflies separate the even and odd spatial symmetries used by the four DCT basis vectors. - output[0] = input[0] + input[3]; - output[1] = input[1] + input[2]; - output[2] = input[1] - input[2]; - output[3] = input[0] - input[3]; - - // Each half-butterfly keeps the optimized kernels' wrapping 32-bit arithmetic before applying the - // normative fixed-point rounding shift. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - - // The staged order groups butterfly partners; AV1 coefficient order interleaves their frequency indices. - output[0] = step[0]; - output[1] = step[2]; - output[2] = step[1]; - output[3] = step[3]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - _ = stageRange; - - // Mirror butterflies separate the even and odd spatial symmetries used by the four DCT basis vectors. - output[0] = input[0] + input[3]; - output[1] = input[1] + input[2]; - output[2] = input[1] - input[2]; - output[3] = input[0] - input[3]; - - // Each half-butterfly keeps the optimized kernels' wrapping 32-bit arithmetic before applying the - // normative fixed-point rounding shift. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - - // The staged order groups butterfly partners; AV1 coefficient order interleaves their frequency indices. - output[0] = step[0]; - output[1] = step[2]; - output[2] = step[1]; - output[3] = step[3]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs index 13e02983e..d45796ddd 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs @@ -6,37 +6,14 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// /// Defines the four-point AV1 forward discrete cosine transform operator. /// -internal readonly partial struct Av1Dct4Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Dct4Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative four-point AV1 forward discrete cosine transform. - /// - /// The four spatial-domain residual values. - /// The four frequency-domain coefficients. - /// The four-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = stageRange; - - // Mirror butterflies separate the even and odd spatial symmetries used by the four DCT basis vectors. - output[0] = input[0] + input[3]; - output[1] = input[1] + input[2]; - output[2] = input[1] - input[2]; - output[3] = input[0] - input[3]; - - // Each half-butterfly widens before multiplication and applies the normative fixed-point rounding shift. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - - // The staged order groups butterfly partners; AV1 coefficient order interleaves their frequency indices. - output[0] = step[0]; - output[1] = step[2]; - output[2] = step[1]; - output[3] = step[3]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct4(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs deleted file mode 100644 index 32d5118dc..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs +++ /dev/null @@ -1,2246 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the sixty-four-point forward DCT operator. -/// -internal readonly partial struct Av1Dct64Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[63]; - output[1] = input[1] + input[62]; - output[2] = input[2] + input[61]; - output[3] = input[3] + input[60]; - output[4] = input[4] + input[59]; - output[5] = input[5] + input[58]; - output[6] = input[6] + input[57]; - output[7] = input[7] + input[56]; - output[8] = input[8] + input[55]; - output[9] = input[9] + input[54]; - output[10] = input[10] + input[53]; - output[11] = input[11] + input[52]; - output[12] = input[12] + input[51]; - output[13] = input[13] + input[50]; - output[14] = input[14] + input[49]; - output[15] = input[15] + input[48]; - output[16] = input[16] + input[47]; - output[17] = input[17] + input[46]; - output[18] = input[18] + input[45]; - output[19] = input[19] + input[44]; - output[20] = input[20] + input[43]; - output[21] = input[21] + input[42]; - output[22] = input[22] + input[41]; - output[23] = input[23] + input[40]; - output[24] = input[24] + input[39]; - output[25] = input[25] + input[38]; - output[26] = input[26] + input[37]; - output[27] = input[27] + input[36]; - output[28] = input[28] + input[35]; - output[29] = input[29] + input[34]; - output[30] = input[30] + input[33]; - output[31] = input[31] + input[32]; - output[32] = -input[32] + input[31]; - output[33] = -input[33] + input[30]; - output[34] = -input[34] + input[29]; - output[35] = -input[35] + input[28]; - output[36] = -input[36] + input[27]; - output[37] = -input[37] + input[26]; - output[38] = -input[38] + input[25]; - output[39] = -input[39] + input[24]; - output[40] = -input[40] + input[23]; - output[41] = -input[41] + input[22]; - output[42] = -input[42] + input[21]; - output[43] = -input[43] + input[20]; - output[44] = -input[44] + input[19]; - output[45] = -input[45] + input[18]; - output[46] = -input[46] + input[17]; - output[47] = -input[47] + input[16]; - output[48] = -input[48] + input[15]; - output[49] = -input[49] + input[14]; - output[50] = -input[50] + input[13]; - output[51] = -input[51] + input[12]; - output[52] = -input[52] + input[11]; - output[53] = -input[53] + input[10]; - output[54] = -input[54] + input[9]; - output[55] = -input[55] + input[8]; - output[56] = -input[56] + input[7]; - output[57] = -input[57] + input[6]; - output[58] = -input[58] + input[5]; - output[59] = -input[59] + input[4]; - output[60] = -input[60] + input[3]; - output[61] = -input[61] + input[2]; - output[62] = -input[62] + input[1]; - output[63] = -input[63] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[31]; - step[1] = output[1] + output[30]; - step[2] = output[2] + output[29]; - step[3] = output[3] + output[28]; - step[4] = output[4] + output[27]; - step[5] = output[5] + output[26]; - step[6] = output[6] + output[25]; - step[7] = output[7] + output[24]; - step[8] = output[8] + output[23]; - step[9] = output[9] + output[22]; - step[10] = output[10] + output[21]; - step[11] = output[11] + output[20]; - step[12] = output[12] + output[19]; - step[13] = output[13] + output[18]; - step[14] = output[14] + output[17]; - step[15] = output[15] + output[16]; - step[16] = -output[16] + output[15]; - step[17] = -output[17] + output[14]; - step[18] = -output[18] + output[13]; - step[19] = -output[19] + output[12]; - step[20] = -output[20] + output[11]; - step[21] = -output[21] + output[10]; - step[22] = -output[22] + output[9]; - step[23] = -output[23] + output[8]; - step[24] = -output[24] + output[7]; - step[25] = -output[25] + output[6]; - step[26] = -output[26] + output[5]; - step[27] = -output[27] + output[4]; - step[28] = -output[28] + output[3]; - step[29] = -output[29] + output[2]; - step[30] = -output[30] + output[1]; - step[31] = -output[31] + output[0]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[48], cospi[32], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[49], cospi[32], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[50], cospi[32], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[51], cospi[32], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[52], cospi[32], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[53], cospi[32], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[54], cospi[32], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[55], cospi[32], output[40], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[15]; - output[1] = step[1] + step[14]; - output[2] = step[2] + step[13]; - output[3] = step[3] + step[12]; - output[4] = step[4] + step[11]; - output[5] = step[5] + step[10]; - output[6] = step[6] + step[9]; - output[7] = step[7] + step[8]; - output[8] = -step[8] + step[7]; - output[9] = -step[9] + step[6]; - output[10] = -step[10] + step[5]; - output[11] = -step[11] + step[4]; - output[12] = -step[12] + step[3]; - output[13] = -step[13] + step[2]; - output[14] = -step[14] + step[1]; - output[15] = -step[15] + step[0]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[24], cospi[32], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[25], cospi[32], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[26], cospi[32], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[27], cospi[32], step[20], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[47]; - output[33] = step[33] + step[46]; - output[34] = step[34] + step[45]; - output[35] = step[35] + step[44]; - output[36] = step[36] + step[43]; - output[37] = step[37] + step[42]; - output[38] = step[38] + step[41]; - output[39] = step[39] + step[40]; - output[40] = -step[40] + step[39]; - output[41] = -step[41] + step[38]; - output[42] = -step[42] + step[37]; - output[43] = -step[43] + step[36]; - output[44] = -step[44] + step[35]; - output[45] = -step[45] + step[34]; - output[46] = -step[46] + step[33]; - output[47] = -step[47] + step[32]; - output[48] = -step[48] + step[63]; - output[49] = -step[49] + step[62]; - output[50] = -step[50] + step[61]; - output[51] = -step[51] + step[60]; - output[52] = -step[52] + step[59]; - output[53] = -step[53] + step[58]; - output[54] = -step[54] + step[57]; - output[55] = -step[55] + step[56]; - output[56] = step[56] + step[55]; - output[57] = step[57] + step[54]; - output[58] = step[58] + step[53]; - output[59] = step[59] + step[52]; - output[60] = step[60] + step[51]; - output[61] = step[61] + step[50]; - output[62] = step[62] + step[49]; - output[63] = step[63] + step[48]; - - // Stage 4 continues the factorization as independent sixteen-sample groups. - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16] + output[23]; - step[17] = output[17] + output[22]; - step[18] = output[18] + output[21]; - step[19] = output[19] + output[20]; - step[20] = -output[20] + output[19]; - step[21] = -output[21] + output[18]; - step[22] = -output[22] + output[17]; - step[23] = -output[23] + output[16]; - step[24] = -output[24] + output[31]; - step[25] = -output[25] + output[30]; - step[26] = -output[26] + output[29]; - step[27] = -output[27] + output[28]; - step[28] = output[28] + output[27]; - step[29] = output[29] + output[26]; - step[30] = output[30] + output[25]; - step[31] = output[31] + output[24]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(cospi[48], output[52], -cospi[16], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[48], output[53], -cospi[16], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[48], output[54], -cospi[16], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[48], output[55], -cospi[16], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[16], output[56], cospi[48], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[16], output[57], cospi[48], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[16], output[58], cospi[48], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[16], output[59], cospi[48], output[36], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(cospi[48], step[26], -cospi[16], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[48], step[27], -cospi[16], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[16], step[28], cospi[48], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[16], step[29], cospi[48], step[18], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[39]; - output[33] = step[33] + step[38]; - output[34] = step[34] + step[37]; - output[35] = step[35] + step[36]; - output[36] = -step[36] + step[35]; - output[37] = -step[37] + step[34]; - output[38] = -step[38] + step[33]; - output[39] = -step[39] + step[32]; - output[40] = -step[40] + step[47]; - output[41] = -step[41] + step[46]; - output[42] = -step[42] + step[45]; - output[43] = -step[43] + step[44]; - output[44] = step[44] + step[43]; - output[45] = step[45] + step[42]; - output[46] = step[46] + step[41]; - output[47] = step[47] + step[40]; - output[48] = step[48] + step[55]; - output[49] = step[49] + step[54]; - output[50] = step[50] + step[53]; - output[51] = step[51] + step[52]; - output[52] = -step[52] + step[51]; - output[53] = -step[53] + step[50]; - output[54] = -step[54] + step[49]; - output[55] = -step[55] + step[48]; - output[56] = -step[56] + step[63]; - output[57] = -step[57] + step[62]; - output[58] = -step[58] + step[61]; - output[59] = -step[59] + step[60]; - output[60] = step[60] + step[59]; - output[61] = step[61] + step[58]; - output[62] = step[62] + step[57]; - output[63] = step[63] + step[56]; - - // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - step[16] = output[16] + output[19]; - step[17] = output[17] + output[18]; - step[18] = -output[18] + output[17]; - step[19] = -output[19] + output[16]; - step[20] = -output[20] + output[23]; - step[21] = -output[21] + output[22]; - step[22] = output[22] + output[21]; - step[23] = output[23] + output[20]; - step[24] = output[24] + output[27]; - step[25] = output[25] + output[26]; - step[26] = -output[26] + output[25]; - step[27] = -output[27] + output[24]; - step[28] = -output[28] + output[31]; - step[29] = -output[29] + output[30]; - step[30] = output[30] + output[29]; - step[31] = output[31] + output[28]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(cospi[24], output[50], -cospi[40], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[24], output[51], -cospi[40], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[40], output[52], cospi[24], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[40], output[53], cospi[24], output[42], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(cospi[56], output[58], -cospi[8], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[56], output[59], -cospi[8], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[8], output[60], cospi[56], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[8], output[61], cospi[56], output[34], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(cospi[24], step[25], -cospi[40], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[40], step[26], cospi[24], step[21], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(cospi[56], step[29], -cospi[8], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[8], step[30], cospi[56], step[17], cosBit); - output[31] = step[31]; - output[32] = step[32] + step[35]; - output[33] = step[33] + step[34]; - output[34] = -step[34] + step[33]; - output[35] = -step[35] + step[32]; - output[36] = -step[36] + step[39]; - output[37] = -step[37] + step[38]; - output[38] = step[38] + step[37]; - output[39] = step[39] + step[36]; - output[40] = step[40] + step[43]; - output[41] = step[41] + step[42]; - output[42] = -step[42] + step[41]; - output[43] = -step[43] + step[40]; - output[44] = -step[44] + step[47]; - output[45] = -step[45] + step[46]; - output[46] = step[46] + step[45]; - output[47] = step[47] + step[44]; - output[48] = step[48] + step[51]; - output[49] = step[49] + step[50]; - output[50] = -step[50] + step[49]; - output[51] = -step[51] + step[48]; - output[52] = -step[52] + step[55]; - output[53] = -step[53] + step[54]; - output[54] = step[54] + step[53]; - output[55] = step[55] + step[52]; - output[56] = step[56] + step[59]; - output[57] = step[57] + step[58]; - output[58] = -step[58] + step[57]; - output[59] = -step[59] + step[56]; - output[60] = -step[60] + step[63]; - output[61] = -step[61] + step[62]; - output[62] = step[62] + step[61]; - output[63] = step[63] + step[60]; - - // Stage 8 applies the next level of odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - step[16] = output[16] + output[17]; - step[17] = -output[17] + output[16]; - step[18] = -output[18] + output[19]; - step[19] = output[19] + output[18]; - step[20] = output[20] + output[21]; - step[21] = -output[21] + output[20]; - step[22] = -output[22] + output[23]; - step[23] = output[23] + output[22]; - step[24] = output[24] + output[25]; - step[25] = -output[25] + output[24]; - step[26] = -output[26] + output[27]; - step[27] = output[27] + output[26]; - step[28] = output[28] + output[29]; - step[29] = -output[29] + output[28]; - step[30] = -output[30] + output[31]; - step[31] = output[31] + output[30]; - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(cospi[12], output[49], -cospi[52], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[52], output[50], cospi[12], output[45], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(cospi[44], output[53], -cospi[20], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[20], output[54], cospi[44], output[41], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(cospi[28], output[57], -cospi[36], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[36], output[58], cospi[28], output[37], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(cospi[60], output[61], -cospi[4], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[4], output[62], cospi[60], output[33], cosBit); - step[63] = output[63]; - - // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[6], step[24], -cospi[58], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[38], step[25], -cospi[26], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[22], step[26], -cospi[42], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[54], step[27], -cospi[10], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[14], step[28], -cospi[50], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[46], step[29], -cospi[18], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[30], step[30], -cospi[34], step[17], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[62], step[31], -cospi[2], step[16], cosBit); - output[32] = step[32] + step[33]; - output[33] = -step[33] + step[32]; - output[34] = -step[34] + step[35]; - output[35] = step[35] + step[34]; - output[36] = step[36] + step[37]; - output[37] = -step[37] + step[36]; - output[38] = -step[38] + step[39]; - output[39] = step[39] + step[38]; - output[40] = step[40] + step[41]; - output[41] = -step[41] + step[40]; - output[42] = -step[42] + step[43]; - output[43] = step[43] + step[42]; - output[44] = step[44] + step[45]; - output[45] = -step[45] + step[44]; - output[46] = -step[46] + step[47]; - output[47] = step[47] + step[46]; - output[48] = step[48] + step[49]; - output[49] = -step[49] + step[48]; - output[50] = -step[50] + step[51]; - output[51] = step[51] + step[50]; - output[52] = step[52] + step[53]; - output[53] = -step[53] + step[52]; - output[54] = -step[54] + step[55]; - output[55] = step[55] + step[54]; - output[56] = step[56] + step[57]; - output[57] = -step[57] + step[56]; - output[58] = -step[58] + step[59]; - output[59] = step[59] + step[58]; - output[60] = step[60] + step[61]; - output[61] = -step[61] + step[60]; - output[62] = -step[62] + step[63]; - output[63] = step[63] + step[62]; - - // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[3], output[48], -cospi[61], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[35], output[49], -cospi[29], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[19], output[50], -cospi[45], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[51], output[51], -cospi[13], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[11], output[52], -cospi[53], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[43], output[53], -cospi[21], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[27], output[54], -cospi[37], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[59], output[55], -cospi[5], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[7], output[56], -cospi[57], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[39], output[57], -cospi[25], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[23], output[58], -cospi[41], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[55], output[59], -cospi[9], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[15], output[60], -cospi[49], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[47], output[61], -cospi[17], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[31], output[62], -cospi[33], output[33], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[63], output[63], -cospi[1], output[32], cosBit); - - // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[32]; - output[2] = step[16]; - output[3] = step[48]; - output[4] = step[8]; - output[5] = step[40]; - output[6] = step[24]; - output[7] = step[56]; - output[8] = step[4]; - output[9] = step[36]; - output[10] = step[20]; - output[11] = step[52]; - output[12] = step[12]; - output[13] = step[44]; - output[14] = step[28]; - output[15] = step[60]; - output[16] = step[2]; - output[17] = step[34]; - output[18] = step[18]; - output[19] = step[50]; - output[20] = step[10]; - output[21] = step[42]; - output[22] = step[26]; - output[23] = step[58]; - output[24] = step[6]; - output[25] = step[38]; - output[26] = step[22]; - output[27] = step[54]; - output[28] = step[14]; - output[29] = step[46]; - output[30] = step[30]; - output[31] = step[62]; - output[32] = step[1]; - output[33] = step[33]; - output[34] = step[17]; - output[35] = step[49]; - output[36] = step[9]; - output[37] = step[41]; - output[38] = step[25]; - output[39] = step[57]; - output[40] = step[5]; - output[41] = step[37]; - output[42] = step[21]; - output[43] = step[53]; - output[44] = step[13]; - output[45] = step[45]; - output[46] = step[29]; - output[47] = step[61]; - output[48] = step[3]; - output[49] = step[35]; - output[50] = step[19]; - output[51] = step[51]; - output[52] = step[11]; - output[53] = step[43]; - output[54] = step[27]; - output[55] = step[59]; - output[56] = step[7]; - output[57] = step[39]; - output[58] = step[23]; - output[59] = step[55]; - output[60] = step[15]; - output[61] = step[47]; - output[62] = step[31]; - output[63] = step[63]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[63]; - output[1] = input[1] + input[62]; - output[2] = input[2] + input[61]; - output[3] = input[3] + input[60]; - output[4] = input[4] + input[59]; - output[5] = input[5] + input[58]; - output[6] = input[6] + input[57]; - output[7] = input[7] + input[56]; - output[8] = input[8] + input[55]; - output[9] = input[9] + input[54]; - output[10] = input[10] + input[53]; - output[11] = input[11] + input[52]; - output[12] = input[12] + input[51]; - output[13] = input[13] + input[50]; - output[14] = input[14] + input[49]; - output[15] = input[15] + input[48]; - output[16] = input[16] + input[47]; - output[17] = input[17] + input[46]; - output[18] = input[18] + input[45]; - output[19] = input[19] + input[44]; - output[20] = input[20] + input[43]; - output[21] = input[21] + input[42]; - output[22] = input[22] + input[41]; - output[23] = input[23] + input[40]; - output[24] = input[24] + input[39]; - output[25] = input[25] + input[38]; - output[26] = input[26] + input[37]; - output[27] = input[27] + input[36]; - output[28] = input[28] + input[35]; - output[29] = input[29] + input[34]; - output[30] = input[30] + input[33]; - output[31] = input[31] + input[32]; - output[32] = -input[32] + input[31]; - output[33] = -input[33] + input[30]; - output[34] = -input[34] + input[29]; - output[35] = -input[35] + input[28]; - output[36] = -input[36] + input[27]; - output[37] = -input[37] + input[26]; - output[38] = -input[38] + input[25]; - output[39] = -input[39] + input[24]; - output[40] = -input[40] + input[23]; - output[41] = -input[41] + input[22]; - output[42] = -input[42] + input[21]; - output[43] = -input[43] + input[20]; - output[44] = -input[44] + input[19]; - output[45] = -input[45] + input[18]; - output[46] = -input[46] + input[17]; - output[47] = -input[47] + input[16]; - output[48] = -input[48] + input[15]; - output[49] = -input[49] + input[14]; - output[50] = -input[50] + input[13]; - output[51] = -input[51] + input[12]; - output[52] = -input[52] + input[11]; - output[53] = -input[53] + input[10]; - output[54] = -input[54] + input[9]; - output[55] = -input[55] + input[8]; - output[56] = -input[56] + input[7]; - output[57] = -input[57] + input[6]; - output[58] = -input[58] + input[5]; - output[59] = -input[59] + input[4]; - output[60] = -input[60] + input[3]; - output[61] = -input[61] + input[2]; - output[62] = -input[62] + input[1]; - output[63] = -input[63] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[31]; - step[1] = output[1] + output[30]; - step[2] = output[2] + output[29]; - step[3] = output[3] + output[28]; - step[4] = output[4] + output[27]; - step[5] = output[5] + output[26]; - step[6] = output[6] + output[25]; - step[7] = output[7] + output[24]; - step[8] = output[8] + output[23]; - step[9] = output[9] + output[22]; - step[10] = output[10] + output[21]; - step[11] = output[11] + output[20]; - step[12] = output[12] + output[19]; - step[13] = output[13] + output[18]; - step[14] = output[14] + output[17]; - step[15] = output[15] + output[16]; - step[16] = -output[16] + output[15]; - step[17] = -output[17] + output[14]; - step[18] = -output[18] + output[13]; - step[19] = -output[19] + output[12]; - step[20] = -output[20] + output[11]; - step[21] = -output[21] + output[10]; - step[22] = -output[22] + output[9]; - step[23] = -output[23] + output[8]; - step[24] = -output[24] + output[7]; - step[25] = -output[25] + output[6]; - step[26] = -output[26] + output[5]; - step[27] = -output[27] + output[4]; - step[28] = -output[28] + output[3]; - step[29] = -output[29] + output[2]; - step[30] = -output[30] + output[1]; - step[31] = -output[31] + output[0]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[48], cospi[32], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[49], cospi[32], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[50], cospi[32], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[51], cospi[32], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[52], cospi[32], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[53], cospi[32], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[54], cospi[32], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[55], cospi[32], output[40], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[15]; - output[1] = step[1] + step[14]; - output[2] = step[2] + step[13]; - output[3] = step[3] + step[12]; - output[4] = step[4] + step[11]; - output[5] = step[5] + step[10]; - output[6] = step[6] + step[9]; - output[7] = step[7] + step[8]; - output[8] = -step[8] + step[7]; - output[9] = -step[9] + step[6]; - output[10] = -step[10] + step[5]; - output[11] = -step[11] + step[4]; - output[12] = -step[12] + step[3]; - output[13] = -step[13] + step[2]; - output[14] = -step[14] + step[1]; - output[15] = -step[15] + step[0]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[24], cospi[32], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[25], cospi[32], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[26], cospi[32], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[27], cospi[32], step[20], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[47]; - output[33] = step[33] + step[46]; - output[34] = step[34] + step[45]; - output[35] = step[35] + step[44]; - output[36] = step[36] + step[43]; - output[37] = step[37] + step[42]; - output[38] = step[38] + step[41]; - output[39] = step[39] + step[40]; - output[40] = -step[40] + step[39]; - output[41] = -step[41] + step[38]; - output[42] = -step[42] + step[37]; - output[43] = -step[43] + step[36]; - output[44] = -step[44] + step[35]; - output[45] = -step[45] + step[34]; - output[46] = -step[46] + step[33]; - output[47] = -step[47] + step[32]; - output[48] = -step[48] + step[63]; - output[49] = -step[49] + step[62]; - output[50] = -step[50] + step[61]; - output[51] = -step[51] + step[60]; - output[52] = -step[52] + step[59]; - output[53] = -step[53] + step[58]; - output[54] = -step[54] + step[57]; - output[55] = -step[55] + step[56]; - output[56] = step[56] + step[55]; - output[57] = step[57] + step[54]; - output[58] = step[58] + step[53]; - output[59] = step[59] + step[52]; - output[60] = step[60] + step[51]; - output[61] = step[61] + step[50]; - output[62] = step[62] + step[49]; - output[63] = step[63] + step[48]; - - // Stage 4 continues the factorization as independent sixteen-sample groups. - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16] + output[23]; - step[17] = output[17] + output[22]; - step[18] = output[18] + output[21]; - step[19] = output[19] + output[20]; - step[20] = -output[20] + output[19]; - step[21] = -output[21] + output[18]; - step[22] = -output[22] + output[17]; - step[23] = -output[23] + output[16]; - step[24] = -output[24] + output[31]; - step[25] = -output[25] + output[30]; - step[26] = -output[26] + output[29]; - step[27] = -output[27] + output[28]; - step[28] = output[28] + output[27]; - step[29] = output[29] + output[26]; - step[30] = output[30] + output[25]; - step[31] = output[31] + output[24]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(cospi[48], output[52], -cospi[16], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[48], output[53], -cospi[16], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[48], output[54], -cospi[16], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[48], output[55], -cospi[16], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[16], output[56], cospi[48], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[16], output[57], cospi[48], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[16], output[58], cospi[48], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[16], output[59], cospi[48], output[36], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(cospi[48], step[26], -cospi[16], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[48], step[27], -cospi[16], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[16], step[28], cospi[48], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[16], step[29], cospi[48], step[18], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[39]; - output[33] = step[33] + step[38]; - output[34] = step[34] + step[37]; - output[35] = step[35] + step[36]; - output[36] = -step[36] + step[35]; - output[37] = -step[37] + step[34]; - output[38] = -step[38] + step[33]; - output[39] = -step[39] + step[32]; - output[40] = -step[40] + step[47]; - output[41] = -step[41] + step[46]; - output[42] = -step[42] + step[45]; - output[43] = -step[43] + step[44]; - output[44] = step[44] + step[43]; - output[45] = step[45] + step[42]; - output[46] = step[46] + step[41]; - output[47] = step[47] + step[40]; - output[48] = step[48] + step[55]; - output[49] = step[49] + step[54]; - output[50] = step[50] + step[53]; - output[51] = step[51] + step[52]; - output[52] = -step[52] + step[51]; - output[53] = -step[53] + step[50]; - output[54] = -step[54] + step[49]; - output[55] = -step[55] + step[48]; - output[56] = -step[56] + step[63]; - output[57] = -step[57] + step[62]; - output[58] = -step[58] + step[61]; - output[59] = -step[59] + step[60]; - output[60] = step[60] + step[59]; - output[61] = step[61] + step[58]; - output[62] = step[62] + step[57]; - output[63] = step[63] + step[56]; - - // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - step[16] = output[16] + output[19]; - step[17] = output[17] + output[18]; - step[18] = -output[18] + output[17]; - step[19] = -output[19] + output[16]; - step[20] = -output[20] + output[23]; - step[21] = -output[21] + output[22]; - step[22] = output[22] + output[21]; - step[23] = output[23] + output[20]; - step[24] = output[24] + output[27]; - step[25] = output[25] + output[26]; - step[26] = -output[26] + output[25]; - step[27] = -output[27] + output[24]; - step[28] = -output[28] + output[31]; - step[29] = -output[29] + output[30]; - step[30] = output[30] + output[29]; - step[31] = output[31] + output[28]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(cospi[24], output[50], -cospi[40], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[24], output[51], -cospi[40], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[40], output[52], cospi[24], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[40], output[53], cospi[24], output[42], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(cospi[56], output[58], -cospi[8], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[56], output[59], -cospi[8], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[8], output[60], cospi[56], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[8], output[61], cospi[56], output[34], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(cospi[24], step[25], -cospi[40], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[40], step[26], cospi[24], step[21], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(cospi[56], step[29], -cospi[8], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[8], step[30], cospi[56], step[17], cosBit); - output[31] = step[31]; - output[32] = step[32] + step[35]; - output[33] = step[33] + step[34]; - output[34] = -step[34] + step[33]; - output[35] = -step[35] + step[32]; - output[36] = -step[36] + step[39]; - output[37] = -step[37] + step[38]; - output[38] = step[38] + step[37]; - output[39] = step[39] + step[36]; - output[40] = step[40] + step[43]; - output[41] = step[41] + step[42]; - output[42] = -step[42] + step[41]; - output[43] = -step[43] + step[40]; - output[44] = -step[44] + step[47]; - output[45] = -step[45] + step[46]; - output[46] = step[46] + step[45]; - output[47] = step[47] + step[44]; - output[48] = step[48] + step[51]; - output[49] = step[49] + step[50]; - output[50] = -step[50] + step[49]; - output[51] = -step[51] + step[48]; - output[52] = -step[52] + step[55]; - output[53] = -step[53] + step[54]; - output[54] = step[54] + step[53]; - output[55] = step[55] + step[52]; - output[56] = step[56] + step[59]; - output[57] = step[57] + step[58]; - output[58] = -step[58] + step[57]; - output[59] = -step[59] + step[56]; - output[60] = -step[60] + step[63]; - output[61] = -step[61] + step[62]; - output[62] = step[62] + step[61]; - output[63] = step[63] + step[60]; - - // Stage 8 applies the next level of odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - step[16] = output[16] + output[17]; - step[17] = -output[17] + output[16]; - step[18] = -output[18] + output[19]; - step[19] = output[19] + output[18]; - step[20] = output[20] + output[21]; - step[21] = -output[21] + output[20]; - step[22] = -output[22] + output[23]; - step[23] = output[23] + output[22]; - step[24] = output[24] + output[25]; - step[25] = -output[25] + output[24]; - step[26] = -output[26] + output[27]; - step[27] = output[27] + output[26]; - step[28] = output[28] + output[29]; - step[29] = -output[29] + output[28]; - step[30] = -output[30] + output[31]; - step[31] = output[31] + output[30]; - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(cospi[12], output[49], -cospi[52], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[52], output[50], cospi[12], output[45], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(cospi[44], output[53], -cospi[20], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[20], output[54], cospi[44], output[41], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(cospi[28], output[57], -cospi[36], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[36], output[58], cospi[28], output[37], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(cospi[60], output[61], -cospi[4], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[4], output[62], cospi[60], output[33], cosBit); - step[63] = output[63]; - - // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[6], step[24], -cospi[58], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[38], step[25], -cospi[26], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[22], step[26], -cospi[42], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[54], step[27], -cospi[10], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[14], step[28], -cospi[50], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[46], step[29], -cospi[18], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[30], step[30], -cospi[34], step[17], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[62], step[31], -cospi[2], step[16], cosBit); - output[32] = step[32] + step[33]; - output[33] = -step[33] + step[32]; - output[34] = -step[34] + step[35]; - output[35] = step[35] + step[34]; - output[36] = step[36] + step[37]; - output[37] = -step[37] + step[36]; - output[38] = -step[38] + step[39]; - output[39] = step[39] + step[38]; - output[40] = step[40] + step[41]; - output[41] = -step[41] + step[40]; - output[42] = -step[42] + step[43]; - output[43] = step[43] + step[42]; - output[44] = step[44] + step[45]; - output[45] = -step[45] + step[44]; - output[46] = -step[46] + step[47]; - output[47] = step[47] + step[46]; - output[48] = step[48] + step[49]; - output[49] = -step[49] + step[48]; - output[50] = -step[50] + step[51]; - output[51] = step[51] + step[50]; - output[52] = step[52] + step[53]; - output[53] = -step[53] + step[52]; - output[54] = -step[54] + step[55]; - output[55] = step[55] + step[54]; - output[56] = step[56] + step[57]; - output[57] = -step[57] + step[56]; - output[58] = -step[58] + step[59]; - output[59] = step[59] + step[58]; - output[60] = step[60] + step[61]; - output[61] = -step[61] + step[60]; - output[62] = -step[62] + step[63]; - output[63] = step[63] + step[62]; - - // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[3], output[48], -cospi[61], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[35], output[49], -cospi[29], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[19], output[50], -cospi[45], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[51], output[51], -cospi[13], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[11], output[52], -cospi[53], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[43], output[53], -cospi[21], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[27], output[54], -cospi[37], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[59], output[55], -cospi[5], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[7], output[56], -cospi[57], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[39], output[57], -cospi[25], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[23], output[58], -cospi[41], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[55], output[59], -cospi[9], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[15], output[60], -cospi[49], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[47], output[61], -cospi[17], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[31], output[62], -cospi[33], output[33], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[63], output[63], -cospi[1], output[32], cosBit); - - // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[32]; - output[2] = step[16]; - output[3] = step[48]; - output[4] = step[8]; - output[5] = step[40]; - output[6] = step[24]; - output[7] = step[56]; - output[8] = step[4]; - output[9] = step[36]; - output[10] = step[20]; - output[11] = step[52]; - output[12] = step[12]; - output[13] = step[44]; - output[14] = step[28]; - output[15] = step[60]; - output[16] = step[2]; - output[17] = step[34]; - output[18] = step[18]; - output[19] = step[50]; - output[20] = step[10]; - output[21] = step[42]; - output[22] = step[26]; - output[23] = step[58]; - output[24] = step[6]; - output[25] = step[38]; - output[26] = step[22]; - output[27] = step[54]; - output[28] = step[14]; - output[29] = step[46]; - output[30] = step[30]; - output[31] = step[62]; - output[32] = step[1]; - output[33] = step[33]; - output[34] = step[17]; - output[35] = step[49]; - output[36] = step[9]; - output[37] = step[41]; - output[38] = step[25]; - output[39] = step[57]; - output[40] = step[5]; - output[41] = step[37]; - output[42] = step[21]; - output[43] = step[53]; - output[44] = step[13]; - output[45] = step[45]; - output[46] = step[29]; - output[47] = step[61]; - output[48] = step[3]; - output[49] = step[35]; - output[50] = step[19]; - output[51] = step[51]; - output[52] = step[11]; - output[53] = step[43]; - output[54] = step[27]; - output[55] = step[59]; - output[56] = step[7]; - output[57] = step[39]; - output[58] = step[23]; - output[59] = step[55]; - output[60] = step[15]; - output[61] = step[47]; - output[62] = step[31]; - output[63] = step[63]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[63]; - output[1] = input[1] + input[62]; - output[2] = input[2] + input[61]; - output[3] = input[3] + input[60]; - output[4] = input[4] + input[59]; - output[5] = input[5] + input[58]; - output[6] = input[6] + input[57]; - output[7] = input[7] + input[56]; - output[8] = input[8] + input[55]; - output[9] = input[9] + input[54]; - output[10] = input[10] + input[53]; - output[11] = input[11] + input[52]; - output[12] = input[12] + input[51]; - output[13] = input[13] + input[50]; - output[14] = input[14] + input[49]; - output[15] = input[15] + input[48]; - output[16] = input[16] + input[47]; - output[17] = input[17] + input[46]; - output[18] = input[18] + input[45]; - output[19] = input[19] + input[44]; - output[20] = input[20] + input[43]; - output[21] = input[21] + input[42]; - output[22] = input[22] + input[41]; - output[23] = input[23] + input[40]; - output[24] = input[24] + input[39]; - output[25] = input[25] + input[38]; - output[26] = input[26] + input[37]; - output[27] = input[27] + input[36]; - output[28] = input[28] + input[35]; - output[29] = input[29] + input[34]; - output[30] = input[30] + input[33]; - output[31] = input[31] + input[32]; - output[32] = -input[32] + input[31]; - output[33] = -input[33] + input[30]; - output[34] = -input[34] + input[29]; - output[35] = -input[35] + input[28]; - output[36] = -input[36] + input[27]; - output[37] = -input[37] + input[26]; - output[38] = -input[38] + input[25]; - output[39] = -input[39] + input[24]; - output[40] = -input[40] + input[23]; - output[41] = -input[41] + input[22]; - output[42] = -input[42] + input[21]; - output[43] = -input[43] + input[20]; - output[44] = -input[44] + input[19]; - output[45] = -input[45] + input[18]; - output[46] = -input[46] + input[17]; - output[47] = -input[47] + input[16]; - output[48] = -input[48] + input[15]; - output[49] = -input[49] + input[14]; - output[50] = -input[50] + input[13]; - output[51] = -input[51] + input[12]; - output[52] = -input[52] + input[11]; - output[53] = -input[53] + input[10]; - output[54] = -input[54] + input[9]; - output[55] = -input[55] + input[8]; - output[56] = -input[56] + input[7]; - output[57] = -input[57] + input[6]; - output[58] = -input[58] + input[5]; - output[59] = -input[59] + input[4]; - output[60] = -input[60] + input[3]; - output[61] = -input[61] + input[2]; - output[62] = -input[62] + input[1]; - output[63] = -input[63] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[31]; - step[1] = output[1] + output[30]; - step[2] = output[2] + output[29]; - step[3] = output[3] + output[28]; - step[4] = output[4] + output[27]; - step[5] = output[5] + output[26]; - step[6] = output[6] + output[25]; - step[7] = output[7] + output[24]; - step[8] = output[8] + output[23]; - step[9] = output[9] + output[22]; - step[10] = output[10] + output[21]; - step[11] = output[11] + output[20]; - step[12] = output[12] + output[19]; - step[13] = output[13] + output[18]; - step[14] = output[14] + output[17]; - step[15] = output[15] + output[16]; - step[16] = -output[16] + output[15]; - step[17] = -output[17] + output[14]; - step[18] = -output[18] + output[13]; - step[19] = -output[19] + output[12]; - step[20] = -output[20] + output[11]; - step[21] = -output[21] + output[10]; - step[22] = -output[22] + output[9]; - step[23] = -output[23] + output[8]; - step[24] = -output[24] + output[7]; - step[25] = -output[25] + output[6]; - step[26] = -output[26] + output[5]; - step[27] = -output[27] + output[4]; - step[28] = -output[28] + output[3]; - step[29] = -output[29] + output[2]; - step[30] = -output[30] + output[1]; - step[31] = -output[31] + output[0]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[48], cospi[32], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[49], cospi[32], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[50], cospi[32], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[51], cospi[32], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[52], cospi[32], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[53], cospi[32], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[54], cospi[32], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[55], cospi[32], output[40], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[15]; - output[1] = step[1] + step[14]; - output[2] = step[2] + step[13]; - output[3] = step[3] + step[12]; - output[4] = step[4] + step[11]; - output[5] = step[5] + step[10]; - output[6] = step[6] + step[9]; - output[7] = step[7] + step[8]; - output[8] = -step[8] + step[7]; - output[9] = -step[9] + step[6]; - output[10] = -step[10] + step[5]; - output[11] = -step[11] + step[4]; - output[12] = -step[12] + step[3]; - output[13] = -step[13] + step[2]; - output[14] = -step[14] + step[1]; - output[15] = -step[15] + step[0]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[24], cospi[32], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[25], cospi[32], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[26], cospi[32], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[27], cospi[32], step[20], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[47]; - output[33] = step[33] + step[46]; - output[34] = step[34] + step[45]; - output[35] = step[35] + step[44]; - output[36] = step[36] + step[43]; - output[37] = step[37] + step[42]; - output[38] = step[38] + step[41]; - output[39] = step[39] + step[40]; - output[40] = -step[40] + step[39]; - output[41] = -step[41] + step[38]; - output[42] = -step[42] + step[37]; - output[43] = -step[43] + step[36]; - output[44] = -step[44] + step[35]; - output[45] = -step[45] + step[34]; - output[46] = -step[46] + step[33]; - output[47] = -step[47] + step[32]; - output[48] = -step[48] + step[63]; - output[49] = -step[49] + step[62]; - output[50] = -step[50] + step[61]; - output[51] = -step[51] + step[60]; - output[52] = -step[52] + step[59]; - output[53] = -step[53] + step[58]; - output[54] = -step[54] + step[57]; - output[55] = -step[55] + step[56]; - output[56] = step[56] + step[55]; - output[57] = step[57] + step[54]; - output[58] = step[58] + step[53]; - output[59] = step[59] + step[52]; - output[60] = step[60] + step[51]; - output[61] = step[61] + step[50]; - output[62] = step[62] + step[49]; - output[63] = step[63] + step[48]; - - // Stage 4 continues the factorization as independent sixteen-sample groups. - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16] + output[23]; - step[17] = output[17] + output[22]; - step[18] = output[18] + output[21]; - step[19] = output[19] + output[20]; - step[20] = -output[20] + output[19]; - step[21] = -output[21] + output[18]; - step[22] = -output[22] + output[17]; - step[23] = -output[23] + output[16]; - step[24] = -output[24] + output[31]; - step[25] = -output[25] + output[30]; - step[26] = -output[26] + output[29]; - step[27] = -output[27] + output[28]; - step[28] = output[28] + output[27]; - step[29] = output[29] + output[26]; - step[30] = output[30] + output[25]; - step[31] = output[31] + output[24]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(cospi[48], output[52], -cospi[16], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[48], output[53], -cospi[16], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[48], output[54], -cospi[16], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[48], output[55], -cospi[16], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[16], output[56], cospi[48], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[16], output[57], cospi[48], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[16], output[58], cospi[48], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[16], output[59], cospi[48], output[36], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(cospi[48], step[26], -cospi[16], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[48], step[27], -cospi[16], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[16], step[28], cospi[48], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[16], step[29], cospi[48], step[18], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[39]; - output[33] = step[33] + step[38]; - output[34] = step[34] + step[37]; - output[35] = step[35] + step[36]; - output[36] = -step[36] + step[35]; - output[37] = -step[37] + step[34]; - output[38] = -step[38] + step[33]; - output[39] = -step[39] + step[32]; - output[40] = -step[40] + step[47]; - output[41] = -step[41] + step[46]; - output[42] = -step[42] + step[45]; - output[43] = -step[43] + step[44]; - output[44] = step[44] + step[43]; - output[45] = step[45] + step[42]; - output[46] = step[46] + step[41]; - output[47] = step[47] + step[40]; - output[48] = step[48] + step[55]; - output[49] = step[49] + step[54]; - output[50] = step[50] + step[53]; - output[51] = step[51] + step[52]; - output[52] = -step[52] + step[51]; - output[53] = -step[53] + step[50]; - output[54] = -step[54] + step[49]; - output[55] = -step[55] + step[48]; - output[56] = -step[56] + step[63]; - output[57] = -step[57] + step[62]; - output[58] = -step[58] + step[61]; - output[59] = -step[59] + step[60]; - output[60] = step[60] + step[59]; - output[61] = step[61] + step[58]; - output[62] = step[62] + step[57]; - output[63] = step[63] + step[56]; - - // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - step[16] = output[16] + output[19]; - step[17] = output[17] + output[18]; - step[18] = -output[18] + output[17]; - step[19] = -output[19] + output[16]; - step[20] = -output[20] + output[23]; - step[21] = -output[21] + output[22]; - step[22] = output[22] + output[21]; - step[23] = output[23] + output[20]; - step[24] = output[24] + output[27]; - step[25] = output[25] + output[26]; - step[26] = -output[26] + output[25]; - step[27] = -output[27] + output[24]; - step[28] = -output[28] + output[31]; - step[29] = -output[29] + output[30]; - step[30] = output[30] + output[29]; - step[31] = output[31] + output[28]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(cospi[24], output[50], -cospi[40], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[24], output[51], -cospi[40], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[40], output[52], cospi[24], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[40], output[53], cospi[24], output[42], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(cospi[56], output[58], -cospi[8], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[56], output[59], -cospi[8], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[8], output[60], cospi[56], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[8], output[61], cospi[56], output[34], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(cospi[24], step[25], -cospi[40], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[40], step[26], cospi[24], step[21], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(cospi[56], step[29], -cospi[8], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[8], step[30], cospi[56], step[17], cosBit); - output[31] = step[31]; - output[32] = step[32] + step[35]; - output[33] = step[33] + step[34]; - output[34] = -step[34] + step[33]; - output[35] = -step[35] + step[32]; - output[36] = -step[36] + step[39]; - output[37] = -step[37] + step[38]; - output[38] = step[38] + step[37]; - output[39] = step[39] + step[36]; - output[40] = step[40] + step[43]; - output[41] = step[41] + step[42]; - output[42] = -step[42] + step[41]; - output[43] = -step[43] + step[40]; - output[44] = -step[44] + step[47]; - output[45] = -step[45] + step[46]; - output[46] = step[46] + step[45]; - output[47] = step[47] + step[44]; - output[48] = step[48] + step[51]; - output[49] = step[49] + step[50]; - output[50] = -step[50] + step[49]; - output[51] = -step[51] + step[48]; - output[52] = -step[52] + step[55]; - output[53] = -step[53] + step[54]; - output[54] = step[54] + step[53]; - output[55] = step[55] + step[52]; - output[56] = step[56] + step[59]; - output[57] = step[57] + step[58]; - output[58] = -step[58] + step[57]; - output[59] = -step[59] + step[56]; - output[60] = -step[60] + step[63]; - output[61] = -step[61] + step[62]; - output[62] = step[62] + step[61]; - output[63] = step[63] + step[60]; - - // Stage 8 applies the next level of odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - step[16] = output[16] + output[17]; - step[17] = -output[17] + output[16]; - step[18] = -output[18] + output[19]; - step[19] = output[19] + output[18]; - step[20] = output[20] + output[21]; - step[21] = -output[21] + output[20]; - step[22] = -output[22] + output[23]; - step[23] = output[23] + output[22]; - step[24] = output[24] + output[25]; - step[25] = -output[25] + output[24]; - step[26] = -output[26] + output[27]; - step[27] = output[27] + output[26]; - step[28] = output[28] + output[29]; - step[29] = -output[29] + output[28]; - step[30] = -output[30] + output[31]; - step[31] = output[31] + output[30]; - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(cospi[12], output[49], -cospi[52], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[52], output[50], cospi[12], output[45], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(cospi[44], output[53], -cospi[20], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[20], output[54], cospi[44], output[41], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(cospi[28], output[57], -cospi[36], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[36], output[58], cospi[28], output[37], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(cospi[60], output[61], -cospi[4], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[4], output[62], cospi[60], output[33], cosBit); - step[63] = output[63]; - - // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[6], step[24], -cospi[58], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[38], step[25], -cospi[26], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[22], step[26], -cospi[42], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[54], step[27], -cospi[10], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[14], step[28], -cospi[50], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[46], step[29], -cospi[18], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[30], step[30], -cospi[34], step[17], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[62], step[31], -cospi[2], step[16], cosBit); - output[32] = step[32] + step[33]; - output[33] = -step[33] + step[32]; - output[34] = -step[34] + step[35]; - output[35] = step[35] + step[34]; - output[36] = step[36] + step[37]; - output[37] = -step[37] + step[36]; - output[38] = -step[38] + step[39]; - output[39] = step[39] + step[38]; - output[40] = step[40] + step[41]; - output[41] = -step[41] + step[40]; - output[42] = -step[42] + step[43]; - output[43] = step[43] + step[42]; - output[44] = step[44] + step[45]; - output[45] = -step[45] + step[44]; - output[46] = -step[46] + step[47]; - output[47] = step[47] + step[46]; - output[48] = step[48] + step[49]; - output[49] = -step[49] + step[48]; - output[50] = -step[50] + step[51]; - output[51] = step[51] + step[50]; - output[52] = step[52] + step[53]; - output[53] = -step[53] + step[52]; - output[54] = -step[54] + step[55]; - output[55] = step[55] + step[54]; - output[56] = step[56] + step[57]; - output[57] = -step[57] + step[56]; - output[58] = -step[58] + step[59]; - output[59] = step[59] + step[58]; - output[60] = step[60] + step[61]; - output[61] = -step[61] + step[60]; - output[62] = -step[62] + step[63]; - output[63] = step[63] + step[62]; - - // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[3], output[48], -cospi[61], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[35], output[49], -cospi[29], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[19], output[50], -cospi[45], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[51], output[51], -cospi[13], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[11], output[52], -cospi[53], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[43], output[53], -cospi[21], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[27], output[54], -cospi[37], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[59], output[55], -cospi[5], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[7], output[56], -cospi[57], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[39], output[57], -cospi[25], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[23], output[58], -cospi[41], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[55], output[59], -cospi[9], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[15], output[60], -cospi[49], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[47], output[61], -cospi[17], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[31], output[62], -cospi[33], output[33], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[63], output[63], -cospi[1], output[32], cosBit); - - // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[32]; - output[2] = step[16]; - output[3] = step[48]; - output[4] = step[8]; - output[5] = step[40]; - output[6] = step[24]; - output[7] = step[56]; - output[8] = step[4]; - output[9] = step[36]; - output[10] = step[20]; - output[11] = step[52]; - output[12] = step[12]; - output[13] = step[44]; - output[14] = step[28]; - output[15] = step[60]; - output[16] = step[2]; - output[17] = step[34]; - output[18] = step[18]; - output[19] = step[50]; - output[20] = step[10]; - output[21] = step[42]; - output[22] = step[26]; - output[23] = step[58]; - output[24] = step[6]; - output[25] = step[38]; - output[26] = step[22]; - output[27] = step[54]; - output[28] = step[14]; - output[29] = step[46]; - output[30] = step[30]; - output[31] = step[62]; - output[32] = step[1]; - output[33] = step[33]; - output[34] = step[17]; - output[35] = step[49]; - output[36] = step[9]; - output[37] = step[41]; - output[38] = step[25]; - output[39] = step[57]; - output[40] = step[5]; - output[41] = step[37]; - output[42] = step[21]; - output[43] = step[53]; - output[44] = step[13]; - output[45] = step[45]; - output[46] = step[29]; - output[47] = step[61]; - output[48] = step[3]; - output[49] = step[35]; - output[50] = step[19]; - output[51] = step[51]; - output[52] = step[11]; - output[53] = step[43]; - output[54] = step[27]; - output[55] = step[59]; - output[56] = step[7]; - output[57] = step[39]; - output[58] = step[23]; - output[59] = step[55]; - output[60] = step[15]; - output[61] = step[47]; - output[62] = step[31]; - output[63] = step[63]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs index 32a3019ca..963e140f9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs @@ -4,749 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 64-point AV1 forward discrete cosine transform operator. +/// Defines the sixty-four-point AV1 forward discrete cosine transform operator. /// -internal readonly partial struct Av1Dct64Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Dct64Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 64-point AV1 forward discrete cosine transform. - /// - /// The 64 spatial-domain residual values. - /// The 64 frequency-domain coefficients. - /// The 64-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[63]; - output[1] = input[1] + input[62]; - output[2] = input[2] + input[61]; - output[3] = input[3] + input[60]; - output[4] = input[4] + input[59]; - output[5] = input[5] + input[58]; - output[6] = input[6] + input[57]; - output[7] = input[7] + input[56]; - output[8] = input[8] + input[55]; - output[9] = input[9] + input[54]; - output[10] = input[10] + input[53]; - output[11] = input[11] + input[52]; - output[12] = input[12] + input[51]; - output[13] = input[13] + input[50]; - output[14] = input[14] + input[49]; - output[15] = input[15] + input[48]; - output[16] = input[16] + input[47]; - output[17] = input[17] + input[46]; - output[18] = input[18] + input[45]; - output[19] = input[19] + input[44]; - output[20] = input[20] + input[43]; - output[21] = input[21] + input[42]; - output[22] = input[22] + input[41]; - output[23] = input[23] + input[40]; - output[24] = input[24] + input[39]; - output[25] = input[25] + input[38]; - output[26] = input[26] + input[37]; - output[27] = input[27] + input[36]; - output[28] = input[28] + input[35]; - output[29] = input[29] + input[34]; - output[30] = input[30] + input[33]; - output[31] = input[31] + input[32]; - output[32] = -input[32] + input[31]; - output[33] = -input[33] + input[30]; - output[34] = -input[34] + input[29]; - output[35] = -input[35] + input[28]; - output[36] = -input[36] + input[27]; - output[37] = -input[37] + input[26]; - output[38] = -input[38] + input[25]; - output[39] = -input[39] + input[24]; - output[40] = -input[40] + input[23]; - output[41] = -input[41] + input[22]; - output[42] = -input[42] + input[21]; - output[43] = -input[43] + input[20]; - output[44] = -input[44] + input[19]; - output[45] = -input[45] + input[18]; - output[46] = -input[46] + input[17]; - output[47] = -input[47] + input[16]; - output[48] = -input[48] + input[15]; - output[49] = -input[49] + input[14]; - output[50] = -input[50] + input[13]; - output[51] = -input[51] + input[12]; - output[52] = -input[52] + input[11]; - output[53] = -input[53] + input[10]; - output[54] = -input[54] + input[9]; - output[55] = -input[55] + input[8]; - output[56] = -input[56] + input[7]; - output[57] = -input[57] + input[6]; - output[58] = -input[58] + input[5]; - output[59] = -input[59] + input[4]; - output[60] = -input[60] + input[3]; - output[61] = -input[61] + input[2]; - output[62] = -input[62] + input[1]; - output[63] = -input[63] + input[0]; - - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[31]; - step[1] = output[1] + output[30]; - step[2] = output[2] + output[29]; - step[3] = output[3] + output[28]; - step[4] = output[4] + output[27]; - step[5] = output[5] + output[26]; - step[6] = output[6] + output[25]; - step[7] = output[7] + output[24]; - step[8] = output[8] + output[23]; - step[9] = output[9] + output[22]; - step[10] = output[10] + output[21]; - step[11] = output[11] + output[20]; - step[12] = output[12] + output[19]; - step[13] = output[13] + output[18]; - step[14] = output[14] + output[17]; - step[15] = output[15] + output[16]; - step[16] = -output[16] + output[15]; - step[17] = -output[17] + output[14]; - step[18] = -output[18] + output[13]; - step[19] = -output[19] + output[12]; - step[20] = -output[20] + output[11]; - step[21] = -output[21] + output[10]; - step[22] = -output[22] + output[9]; - step[23] = -output[23] + output[8]; - step[24] = -output[24] + output[7]; - step[25] = -output[25] + output[6]; - step[26] = -output[26] + output[5]; - step[27] = -output[27] + output[4]; - step[28] = -output[28] + output[3]; - step[29] = -output[29] + output[2]; - step[30] = -output[30] + output[1]; - step[31] = -output[31] + output[0]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[48], cospi[32], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[49], cospi[32], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[50], cospi[32], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[51], cospi[32], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[52], cospi[32], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[53], cospi[32], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[54], cospi[32], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[55], cospi[32], output[40], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - output[0] = step[0] + step[15]; - output[1] = step[1] + step[14]; - output[2] = step[2] + step[13]; - output[3] = step[3] + step[12]; - output[4] = step[4] + step[11]; - output[5] = step[5] + step[10]; - output[6] = step[6] + step[9]; - output[7] = step[7] + step[8]; - output[8] = -step[8] + step[7]; - output[9] = -step[9] + step[6]; - output[10] = -step[10] + step[5]; - output[11] = -step[11] + step[4]; - output[12] = -step[12] + step[3]; - output[13] = -step[13] + step[2]; - output[14] = -step[14] + step[1]; - output[15] = -step[15] + step[0]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[24], cospi[32], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[25], cospi[32], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[26], cospi[32], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[27], cospi[32], step[20], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[47]; - output[33] = step[33] + step[46]; - output[34] = step[34] + step[45]; - output[35] = step[35] + step[44]; - output[36] = step[36] + step[43]; - output[37] = step[37] + step[42]; - output[38] = step[38] + step[41]; - output[39] = step[39] + step[40]; - output[40] = -step[40] + step[39]; - output[41] = -step[41] + step[38]; - output[42] = -step[42] + step[37]; - output[43] = -step[43] + step[36]; - output[44] = -step[44] + step[35]; - output[45] = -step[45] + step[34]; - output[46] = -step[46] + step[33]; - output[47] = -step[47] + step[32]; - output[48] = -step[48] + step[63]; - output[49] = -step[49] + step[62]; - output[50] = -step[50] + step[61]; - output[51] = -step[51] + step[60]; - output[52] = -step[52] + step[59]; - output[53] = -step[53] + step[58]; - output[54] = -step[54] + step[57]; - output[55] = -step[55] + step[56]; - output[56] = step[56] + step[55]; - output[57] = step[57] + step[54]; - output[58] = step[58] + step[53]; - output[59] = step[59] + step[52]; - output[60] = step[60] + step[51]; - output[61] = step[61] + step[50]; - output[62] = step[62] + step[49]; - output[63] = step[63] + step[48]; - - // Stage 4 continues the factorization as independent sixteen-sample groups. - step[0] = output[0] + output[7]; - step[1] = output[1] + output[6]; - step[2] = output[2] + output[5]; - step[3] = output[3] + output[4]; - step[4] = -output[4] + output[3]; - step[5] = -output[5] + output[2]; - step[6] = -output[6] + output[1]; - step[7] = -output[7] + output[0]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16] + output[23]; - step[17] = output[17] + output[22]; - step[18] = output[18] + output[21]; - step[19] = output[19] + output[20]; - step[20] = -output[20] + output[19]; - step[21] = -output[21] + output[18]; - step[22] = -output[22] + output[17]; - step[23] = -output[23] + output[16]; - step[24] = -output[24] + output[31]; - step[25] = -output[25] + output[30]; - step[26] = -output[26] + output[29]; - step[27] = -output[27] + output[28]; - step[28] = output[28] + output[27]; - step[29] = output[29] + output[26]; - step[30] = output[30] + output[25]; - step[31] = output[31] + output[24]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(cospi[48], output[52], -cospi[16], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[48], output[53], -cospi[16], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[48], output[54], -cospi[16], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[48], output[55], -cospi[16], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[16], output[56], cospi[48], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[16], output[57], cospi[48], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[16], output[58], cospi[48], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[16], output[59], cospi[48], output[36], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. - output[0] = step[0] + step[3]; - output[1] = step[1] + step[2]; - output[2] = -step[2] + step[1]; - output[3] = -step[3] + step[0]; - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); - output[7] = step[7]; - output[8] = step[8] + step[11]; - output[9] = step[9] + step[10]; - output[10] = -step[10] + step[9]; - output[11] = -step[11] + step[8]; - output[12] = -step[12] + step[15]; - output[13] = -step[13] + step[14]; - output[14] = step[14] + step[13]; - output[15] = step[15] + step[12]; - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(cospi[48], step[26], -cospi[16], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[48], step[27], -cospi[16], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[16], step[28], cospi[48], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[16], step[29], cospi[48], step[18], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = step[32] + step[39]; - output[33] = step[33] + step[38]; - output[34] = step[34] + step[37]; - output[35] = step[35] + step[36]; - output[36] = -step[36] + step[35]; - output[37] = -step[37] + step[34]; - output[38] = -step[38] + step[33]; - output[39] = -step[39] + step[32]; - output[40] = -step[40] + step[47]; - output[41] = -step[41] + step[46]; - output[42] = -step[42] + step[45]; - output[43] = -step[43] + step[44]; - output[44] = step[44] + step[43]; - output[45] = step[45] + step[42]; - output[46] = step[46] + step[41]; - output[47] = step[47] + step[40]; - output[48] = step[48] + step[55]; - output[49] = step[49] + step[54]; - output[50] = step[50] + step[53]; - output[51] = step[51] + step[52]; - output[52] = -step[52] + step[51]; - output[53] = -step[53] + step[50]; - output[54] = -step[54] + step[49]; - output[55] = -step[55] + step[48]; - output[56] = -step[56] + step[63]; - output[57] = -step[57] + step[62]; - output[58] = -step[58] + step[61]; - output[59] = -step[59] + step[60]; - output[60] = step[60] + step[59]; - output[61] = step[61] + step[58]; - output[62] = step[62] + step[57]; - output[63] = step[63] + step[56]; - - // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); - step[4] = output[4] + output[5]; - step[5] = -output[5] + output[4]; - step[6] = -output[6] + output[7]; - step[7] = output[7] + output[6]; - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); - step[15] = output[15]; - step[16] = output[16] + output[19]; - step[17] = output[17] + output[18]; - step[18] = -output[18] + output[17]; - step[19] = -output[19] + output[16]; - step[20] = -output[20] + output[23]; - step[21] = -output[21] + output[22]; - step[22] = output[22] + output[21]; - step[23] = output[23] + output[20]; - step[24] = output[24] + output[27]; - step[25] = output[25] + output[26]; - step[26] = -output[26] + output[25]; - step[27] = -output[27] + output[24]; - step[28] = -output[28] + output[31]; - step[29] = -output[29] + output[30]; - step[30] = output[30] + output[29]; - step[31] = output[31] + output[28]; - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(cospi[24], output[50], -cospi[40], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[24], output[51], -cospi[40], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[40], output[52], cospi[24], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[40], output[53], cospi[24], output[42], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(cospi[56], output[58], -cospi[8], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[56], output[59], -cospi[8], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[8], output[60], cospi[56], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[8], output[61], cospi[56], output[34], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); - output[8] = step[8] + step[9]; - output[9] = -step[9] + step[8]; - output[10] = -step[10] + step[11]; - output[11] = step[11] + step[10]; - output[12] = step[12] + step[13]; - output[13] = -step[13] + step[12]; - output[14] = -step[14] + step[15]; - output[15] = step[15] + step[14]; - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(cospi[24], step[25], -cospi[40], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[40], step[26], cospi[24], step[21], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(cospi[56], step[29], -cospi[8], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[8], step[30], cospi[56], step[17], cosBit); - output[31] = step[31]; - output[32] = step[32] + step[35]; - output[33] = step[33] + step[34]; - output[34] = -step[34] + step[33]; - output[35] = -step[35] + step[32]; - output[36] = -step[36] + step[39]; - output[37] = -step[37] + step[38]; - output[38] = step[38] + step[37]; - output[39] = step[39] + step[36]; - output[40] = step[40] + step[43]; - output[41] = step[41] + step[42]; - output[42] = -step[42] + step[41]; - output[43] = -step[43] + step[40]; - output[44] = -step[44] + step[47]; - output[45] = -step[45] + step[46]; - output[46] = step[46] + step[45]; - output[47] = step[47] + step[44]; - output[48] = step[48] + step[51]; - output[49] = step[49] + step[50]; - output[50] = -step[50] + step[49]; - output[51] = -step[51] + step[48]; - output[52] = -step[52] + step[55]; - output[53] = -step[53] + step[54]; - output[54] = step[54] + step[53]; - output[55] = step[55] + step[52]; - output[56] = step[56] + step[59]; - output[57] = step[57] + step[58]; - output[58] = -step[58] + step[57]; - output[59] = -step[59] + step[56]; - output[60] = -step[60] + step[63]; - output[61] = -step[61] + step[62]; - output[62] = step[62] + step[61]; - output[63] = step[63] + step[60]; - - // Stage 8 applies the next level of odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); - step[16] = output[16] + output[17]; - step[17] = -output[17] + output[16]; - step[18] = -output[18] + output[19]; - step[19] = output[19] + output[18]; - step[20] = output[20] + output[21]; - step[21] = -output[21] + output[20]; - step[22] = -output[22] + output[23]; - step[23] = output[23] + output[22]; - step[24] = output[24] + output[25]; - step[25] = -output[25] + output[24]; - step[26] = -output[26] + output[27]; - step[27] = output[27] + output[26]; - step[28] = output[28] + output[29]; - step[29] = -output[29] + output[28]; - step[30] = -output[30] + output[31]; - step[31] = output[31] + output[30]; - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(cospi[12], output[49], -cospi[52], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[52], output[50], cospi[12], output[45], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(cospi[44], output[53], -cospi[20], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[20], output[54], cospi[44], output[41], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(cospi[28], output[57], -cospi[36], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[36], output[58], cospi[28], output[37], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(cospi[60], output[61], -cospi[4], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[4], output[62], cospi[60], output[33], cosBit); - step[63] = output[63]; - - // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[6], step[24], -cospi[58], step[23], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[38], step[25], -cospi[26], step[22], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[22], step[26], -cospi[42], step[21], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[54], step[27], -cospi[10], step[20], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[14], step[28], -cospi[50], step[19], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[46], step[29], -cospi[18], step[18], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[30], step[30], -cospi[34], step[17], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[62], step[31], -cospi[2], step[16], cosBit); - output[32] = step[32] + step[33]; - output[33] = -step[33] + step[32]; - output[34] = -step[34] + step[35]; - output[35] = step[35] + step[34]; - output[36] = step[36] + step[37]; - output[37] = -step[37] + step[36]; - output[38] = -step[38] + step[39]; - output[39] = step[39] + step[38]; - output[40] = step[40] + step[41]; - output[41] = -step[41] + step[40]; - output[42] = -step[42] + step[43]; - output[43] = step[43] + step[42]; - output[44] = step[44] + step[45]; - output[45] = -step[45] + step[44]; - output[46] = -step[46] + step[47]; - output[47] = step[47] + step[46]; - output[48] = step[48] + step[49]; - output[49] = -step[49] + step[48]; - output[50] = -step[50] + step[51]; - output[51] = step[51] + step[50]; - output[52] = step[52] + step[53]; - output[53] = -step[53] + step[52]; - output[54] = -step[54] + step[55]; - output[55] = step[55] + step[54]; - output[56] = step[56] + step[57]; - output[57] = -step[57] + step[56]; - output[58] = -step[58] + step[59]; - output[59] = step[59] + step[58]; - output[60] = step[60] + step[61]; - output[61] = -step[61] + step[60]; - output[62] = -step[62] + step[63]; - output[63] = step[63] + step[62]; - - // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[3], output[48], -cospi[61], output[47], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[35], output[49], -cospi[29], output[46], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[19], output[50], -cospi[45], output[45], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[51], output[51], -cospi[13], output[44], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[11], output[52], -cospi[53], output[43], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[43], output[53], -cospi[21], output[42], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[27], output[54], -cospi[37], output[41], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[59], output[55], -cospi[5], output[40], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[7], output[56], -cospi[57], output[39], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[39], output[57], -cospi[25], output[38], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[23], output[58], -cospi[41], output[37], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[55], output[59], -cospi[9], output[36], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[15], output[60], -cospi[49], output[35], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[47], output[61], -cospi[17], output[34], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[31], output[62], -cospi[33], output[33], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[63], output[63], -cospi[1], output[32], cosBit); - - // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[32]; - output[2] = step[16]; - output[3] = step[48]; - output[4] = step[8]; - output[5] = step[40]; - output[6] = step[24]; - output[7] = step[56]; - output[8] = step[4]; - output[9] = step[36]; - output[10] = step[20]; - output[11] = step[52]; - output[12] = step[12]; - output[13] = step[44]; - output[14] = step[28]; - output[15] = step[60]; - output[16] = step[2]; - output[17] = step[34]; - output[18] = step[18]; - output[19] = step[50]; - output[20] = step[10]; - output[21] = step[42]; - output[22] = step[26]; - output[23] = step[58]; - output[24] = step[6]; - output[25] = step[38]; - output[26] = step[22]; - output[27] = step[54]; - output[28] = step[14]; - output[29] = step[46]; - output[30] = step[30]; - output[31] = step[62]; - output[32] = step[1]; - output[33] = step[33]; - output[34] = step[17]; - output[35] = step[49]; - output[36] = step[9]; - output[37] = step[41]; - output[38] = step[25]; - output[39] = step[57]; - output[40] = step[5]; - output[41] = step[37]; - output[42] = step[21]; - output[43] = step[53]; - output[44] = step[13]; - output[45] = step[45]; - output[46] = step[29]; - output[47] = step[61]; - output[48] = step[3]; - output[49] = step[35]; - output[50] = step[19]; - output[51] = step[51]; - output[52] = step[11]; - output[53] = step[43]; - output[54] = step[27]; - output[55] = step[59]; - output[56] = step[7]; - output[57] = step[39]; - output[58] = step[23]; - output[59] = step[55]; - output[60] = step[15]; - output[61] = step[47]; - output[62] = step[31]; - output[63] = step[63]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct64(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs deleted file mode 100644 index a5ebfca18..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the eight-point forward DCT operator. -/// -internal readonly partial struct Av1Dct8Forward1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[7]; - output[1] = input[1] + input[6]; - output[2] = input[2] + input[5]; - output[3] = input[3] + input[4]; - output[4] = -input[4] + input[3]; - output[5] = -input[5] + input[2]; - output[6] = -input[6] + input[1]; - output[7] = -input[7] + input[0]; - - // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - - // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - - // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - - // Stage 5 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[4]; - output[2] = step[2]; - output[3] = step[6]; - output[4] = step[1]; - output[5] = step[5]; - output[6] = step[3]; - output[7] = step[7]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[7]; - output[1] = input[1] + input[6]; - output[2] = input[2] + input[5]; - output[3] = input[3] + input[4]; - output[4] = -input[4] + input[3]; - output[5] = -input[5] + input[2]; - output[6] = -input[6] + input[1]; - output[7] = -input[7] + input[0]; - - // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - - // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - - // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - - // Stage 5 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[4]; - output[2] = step[2]; - output[3] = step[6]; - output[4] = step[1]; - output[5] = step[5]; - output[6] = step[3]; - output[7] = step[7]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[7]; - output[1] = input[1] + input[6]; - output[2] = input[2] + input[5]; - output[3] = input[3] + input[4]; - output[4] = -input[4] + input[3]; - output[5] = -input[5] + input[2]; - output[6] = -input[6] + input[1]; - output[7] = -input[7] + input[0]; - - // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - - // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - - // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - - // Stage 5 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[4]; - output[2] = step[2]; - output[3] = step[6]; - output[4] = step[1]; - output[5] = step[5]; - output[6] = step[3]; - output[7] = step[7]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs index 9ce6f7110..3d002d6ac 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs @@ -6,71 +6,14 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// /// Defines the eight-point AV1 forward discrete cosine transform operator. /// -internal readonly partial struct Av1Dct8Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Dct8Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative eight-point AV1 forward discrete cosine transform. - /// - /// The eight spatial-domain residual values. - /// The eight frequency-domain coefficients. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on - // the ranges already established from the coded bit depth and the normative two-dimensional shifts. - _ = stageRange; - - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - output[0] = input[0] + input[7]; - output[1] = input[1] + input[6]; - output[2] = input[2] + input[5]; - output[3] = input[3] + input[4]; - output[4] = -input[4] + input[3]; - output[5] = -input[5] + input[2]; - output[6] = -input[6] + input[1]; - output[7] = -input[7] + input[0]; - - // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = output[0] + output[3]; - step[1] = output[1] + output[2]; - step[2] = -output[2] + output[1]; - step[3] = -output[3] + output[0]; - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); - step[7] = output[7]; - - // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); - output[4] = step[4] + step[5]; - output[5] = -step[5] + step[4]; - output[6] = -step[6] + step[7]; - output[7] = step[7] + step[6]; - - // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); - - // Stage 5 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[4]; - output[2] = step[2]; - output[3] = step[6]; - output[4] = step[1]; - output[5] = step[5]; - output[6] = step[3]; - output[7] = step[7]; - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct8(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformArithmetic.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformArithmetic.cs new file mode 100644 index 000000000..341d62100 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformArithmetic.cs @@ -0,0 +1,848 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Provides the sample-type and vector-width arithmetic used by the shared AV1 forward-transform stage networks. +/// +/// The scalar or SIMD value containing independent transform axes. +internal static class Av1ForwardTransformArithmetic + where TValue : struct +{ + /// + /// Creates the rounding value used by fixed-point transform multiplications. + /// + /// The number of fractional bits in the transform constants. + /// The rounding value in the widened lane shape used by . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Av1TransformRounding CreateRounding(int cosBit) + { + int value = 1 << (cosBit - 1); + Av1TransformRounding rounding = default; + + if (typeof(TValue) == typeof(Vector128) || typeof(TValue) == typeof(Vector128)) + { + rounding.Vector128 = Vector128.Create(value); + } + else if (typeof(TValue) == typeof(Vector256) || typeof(TValue) == typeof(Vector256)) + { + rounding.Vector256 = Vector256.Create(value); + } + else if (typeof(TValue) == typeof(Vector512) || typeof(TValue) == typeof(Vector512)) + { + rounding.Vector512 = Vector512.Create(value); + } + else + { + rounding.Scalar = value; + } + + return rounding; + } + + /// + /// Adds two transform values using the lane arithmetic required by the selected sample type. + /// + /// The left operand. + /// The right operand. + /// The lane-wise sum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue Add(TValue left, TValue right) + { + if (typeof(TValue) == typeof(short)) + { + int resultValue = As(left) + As(right); + short result = (short)Math.Clamp(resultValue, short.MinValue, short.MaxValue); + return As(result); + } + + if (typeof(TValue) == typeof(int)) + { + int result = As(left) + As(right); + return As(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = Vector128.AddSaturate(As>(left), As>(right)); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = As>(left) + As>(right); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = Vector256.AddSaturate(As>(left), As>(right)); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = As>(left) + As>(right); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 result = Vector512.AddSaturate(As>(left), As>(right)); + return As, TValue>(result); + } + + Vector512 vector = As>(left) + As>(right); + return As, TValue>(vector); + } + + /// + /// Subtracts one transform value from another using the lane arithmetic required by the selected sample type. + /// + /// The left operand. + /// The right operand. + /// The lane-wise difference. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue Subtract(TValue left, TValue right) + { + if (typeof(TValue) == typeof(short)) + { + int resultValue = As(left) - As(right); + short result = (short)Math.Clamp(resultValue, short.MinValue, short.MaxValue); + return As(result); + } + + if (typeof(TValue) == typeof(int)) + { + int result = As(left) - As(right); + return As(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = Vector128.SubtractSaturate(As>(left), As>(right)); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = As>(left) - As>(right); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = Vector256.SubtractSaturate(As>(left), As>(right)); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = As>(left) - As>(right); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 result = Vector512.SubtractSaturate(As>(left), As>(right)); + return As, TValue>(result); + } + + Vector512 vector = As>(left) - As>(right); + return As, TValue>(vector); + } + + /// + /// Negates a transform value using wrapping lane arithmetic. + /// + /// The value to negate. + /// The lane-wise negated value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue Negate(TValue value) + { + if (typeof(TValue) == typeof(short)) + { + short result = unchecked((short)-As(value)); + return As(result); + } + + if (typeof(TValue) == typeof(int)) + { + int result = -As(value); + return As(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = Vector128.Zero - As>(value); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = -As>(value); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = Vector256.Zero - As>(value); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = -As>(value); + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 result = Vector512.Zero - As>(value); + return As, TValue>(result); + } + + Vector512 vector = -As>(value); + return As, TValue>(vector); + } + + /// + /// Adds and subtracts two transform values, saturating only the signed sixteen-bit representations. + /// + /// The left operand. + /// The right operand. + /// The lane-wise sum. + /// The lane-wise difference. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void AddSubtract(TValue left, TValue right, out TValue sum, out TValue difference) + { + if (typeof(TValue) == typeof(short)) + { + int leftValue = As(left); + int rightValue = As(right); + short sumValue = (short)Math.Clamp(leftValue + rightValue, short.MinValue, short.MaxValue); + short differenceValue = (short)Math.Clamp(leftValue - rightValue, short.MinValue, short.MaxValue); + + sum = As(sumValue); + difference = As(differenceValue); + return; + } + + if (typeof(TValue) == typeof(int)) + { + int leftValue = As(left); + int rightValue = As(right); + int sumValue = leftValue + rightValue; + int differenceValue = leftValue - rightValue; + + sum = As(sumValue); + difference = As(differenceValue); + return; + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 leftValue = As>(left); + Vector128 rightValue = As>(right); + Vector128 sumValue = Vector128.AddSaturate(leftValue, rightValue); + Vector128 differenceValue = Vector128.SubtractSaturate(leftValue, rightValue); + + sum = As, TValue>(sumValue); + difference = As, TValue>(differenceValue); + return; + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 leftValue = As>(left); + Vector128 rightValue = As>(right); + + sum = As, TValue>(leftValue + rightValue); + difference = As, TValue>(leftValue - rightValue); + return; + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 leftValue = As>(left); + Vector256 rightValue = As>(right); + Vector256 sumValue = Vector256.AddSaturate(leftValue, rightValue); + Vector256 differenceValue = Vector256.SubtractSaturate(leftValue, rightValue); + + sum = As, TValue>(sumValue); + difference = As, TValue>(differenceValue); + return; + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 leftValue = As>(left); + Vector256 rightValue = As>(right); + + sum = As, TValue>(leftValue + rightValue); + difference = As, TValue>(leftValue - rightValue); + return; + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 leftValue = As>(left); + Vector512 rightValue = As>(right); + Vector512 sumValue = Vector512.AddSaturate(leftValue, rightValue); + Vector512 differenceValue = Vector512.SubtractSaturate(leftValue, rightValue); + + sum = As, TValue>(sumValue); + difference = As, TValue>(differenceValue); + return; + } + + Vector512 leftVector = As>(left); + Vector512 rightVector = As>(right); + + sum = As, TValue>(leftVector + rightVector); + difference = As, TValue>(leftVector - rightVector); + } + + /// + /// Calculates both outputs of a rounded, weighted two-input butterfly. + /// + /// The first fixed-point weight. + /// The second fixed-point weight. + /// The first transform value. + /// The second transform value. + /// The first rounded result. + /// The second rounded result. + /// The number of fractional bits in each weight. + /// The rounding value created for this transform. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Butterfly( + int weight0, + int weight1, + TValue input0, + TValue input1, + out TValue output0, + out TValue output1, + int cosBit, + in Av1TransformRounding rounding) + { + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 left = As>(input0); + Vector128 right = As>(input1); + + Av1Transform1dMath.Butterfly(weight0, weight1, in left, in right, out Vector128 result0, out Vector128 result1, cosBit, in rounding.Vector128); + output0 = As, TValue>(result0); + output1 = As, TValue>(result1); + return; + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 left = As>(input0); + Vector256 right = As>(input1); + + Av1Transform1dMath.Butterfly(weight0, weight1, in left, in right, out Vector256 result0, out Vector256 result1, cosBit, in rounding.Vector256); + output0 = As, TValue>(result0); + output1 = As, TValue>(result1); + return; + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 left = As>(input0); + Vector512 right = As>(input1); + + Av1Transform1dMath.Butterfly(weight0, weight1, in left, in right, out Vector512 result0, out Vector512 result1, cosBit, in rounding.Vector512); + output0 = As, TValue>(result0); + output1 = As, TValue>(result1); + return; + } + + output0 = HalfButterfly(weight0, input0, weight1, input1, cosBit, in rounding); + output1 = HalfButterfly(weight1, input0, -weight0, input1, cosBit, in rounding); + } + + /// + /// Calculates one output of a rounded, weighted two-input butterfly. + /// + /// The first fixed-point weight. + /// The first transform value. + /// The second fixed-point weight. + /// The second transform value. + /// The number of fractional bits in each weight. + /// The rounding value created for this transform. + /// The rounded lane-wise weighted sum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue HalfButterfly( + int weight0, + TValue input0, + int weight1, + TValue input1, + int cosBit, + in Av1TransformRounding rounding) + { + if (typeof(TValue) == typeof(short)) + { + int weighted = (weight0 * As(input0)) + (weight1 * As(input1)); + short result = (short)Math.Clamp((weighted + rounding.Scalar) >> cosBit, short.MinValue, short.MaxValue); + return As(result); + } + + if (typeof(TValue) == typeof(int)) + { + int weighted = (weight0 * As(input0)) + (weight1 * As(input1)); + int result = (weighted + rounding.Scalar) >> cosBit; + return As(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = MultiplyRound( + As>(input0), + weight0, + As>(input1), + weight1, + cosBit, + rounding.Vector128); + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = ((As>(input0) * Vector128.Create(weight0)) + + (As>(input1) * Vector128.Create(weight1)) + + rounding.Vector128) >> cosBit; + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = MultiplyRound( + As>(input0), + weight0, + As>(input1), + weight1, + cosBit, + rounding.Vector256); + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = ((As>(input0) * Vector256.Create(weight0)) + + (As>(input1) * Vector256.Create(weight1)) + + rounding.Vector256) >> cosBit; + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 result = MultiplyRound( + As>(input0), + weight0, + As>(input1), + weight1, + cosBit, + rounding.Vector512); + + return As, TValue>(result); + } + + Vector512 vector = ((As>(input0) * Vector512.Create(weight0)) + + (As>(input1) * Vector512.Create(weight1)) + + rounding.Vector512) >> cosBit; + + return As, TValue>(vector); + } + + /// + /// Multiplies a transform value by a fixed-point constant and applies the requested rounding shift. + /// + /// The transform value. + /// The fixed-point multiplier. + /// The number of fractional bits in the multiplier. + /// The rounded lane-wise product. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue MultiplyRound(TValue value, int multiplier, int shift) + { + Av1TransformRounding rounding = CreateRounding(shift); + return HalfButterfly(multiplier, value, 0, default, shift, in rounding); + } + + /// + /// Shifts each transform lane left without saturation. + /// + /// The transform value. + /// The shift count. + /// The shifted lane values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue ShiftLeft(TValue value, int count) + { + if (typeof(TValue) == typeof(short)) + { + short result = unchecked((short)(As(value) << count)); + return As(result); + } + + if (typeof(TValue) == typeof(int)) + { + int result = As(value) << count; + return As(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = As>(value) << count; + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = As>(value) << count; + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = As>(value) << count; + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = As>(value) << count; + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 result = As>(value) << count; + return As, TValue>(result); + } + + Vector512 vector = As>(value) << count; + return As, TValue>(vector); + } + + /// + /// Calculates one rounded sum containing four independently weighted transform values. + /// + /// The first fixed-point weight. + /// The first transform value. + /// The second fixed-point weight. + /// The second transform value. + /// The third fixed-point weight. + /// The third transform value. + /// The fourth fixed-point weight. + /// The fourth transform value. + /// The number of fractional bits in each weight. + /// The rounding value created for this transform. + /// The rounded lane-wise weighted sum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue MultiplyAddRound( + int weight0, + TValue input0, + int weight1, + TValue input1, + int weight2, + TValue input2, + int weight3, + TValue input3, + int cosBit, + in Av1TransformRounding rounding) + { + if (typeof(TValue) == typeof(short)) + { + int weighted = (weight0 * As(input0)) + + (weight1 * As(input1)) + + (weight2 * As(input2)) + + (weight3 * As(input3)); + + short result = (short)Math.Clamp((weighted + rounding.Scalar) >> cosBit, short.MinValue, short.MaxValue); + return As(result); + } + + if (typeof(TValue) == typeof(int)) + { + int weighted = (weight0 * As(input0)) + + (weight1 * As(input1)) + + (weight2 * As(input2)) + + (weight3 * As(input3)); + + int result = (weighted + rounding.Scalar) >> cosBit; + return As(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = MultiplyRound( + As>(input0), + weight0, + As>(input1), + weight1, + As>(input2), + weight2, + As>(input3), + weight3, + cosBit, + rounding.Vector128); + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector128)) + { + Vector128 result = ((As>(input0) * Vector128.Create(weight0)) + + (As>(input1) * Vector128.Create(weight1)) + + (As>(input2) * Vector128.Create(weight2)) + + (As>(input3) * Vector128.Create(weight3)) + + rounding.Vector128) >> cosBit; + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = MultiplyRound( + As>(input0), + weight0, + As>(input1), + weight1, + As>(input2), + weight2, + As>(input3), + weight3, + cosBit, + rounding.Vector256); + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector256)) + { + Vector256 result = ((As>(input0) * Vector256.Create(weight0)) + + (As>(input1) * Vector256.Create(weight1)) + + (As>(input2) * Vector256.Create(weight2)) + + (As>(input3) * Vector256.Create(weight3)) + + rounding.Vector256) >> cosBit; + + return As, TValue>(result); + } + + if (typeof(TValue) == typeof(Vector512)) + { + Vector512 result = MultiplyRound( + As>(input0), + weight0, + As>(input1), + weight1, + As>(input2), + weight2, + As>(input3), + weight3, + cosBit, + rounding.Vector512); + + return As, TValue>(result); + } + + Vector512 vector = ((As>(input0) * Vector512.Create(weight0)) + + (As>(input1) * Vector512.Create(weight1)) + + (As>(input2) * Vector512.Create(weight2)) + + (As>(input3) * Vector512.Create(weight3)) + + rounding.Vector512) >> cosBit; + + return As, TValue>(vector); + } + + /// + /// Converts one value type to another equal-sized value type without changing its bits. + /// + /// The source value type. + /// The destination value type. + /// The value to reinterpret. + /// The reinterpreted value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TTo As(TFrom value) + where TFrom : struct + where TTo : struct + => Unsafe.As(ref value); + + /// + /// Calculates and narrows two weighted 128-bit signed sixteen-bit vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MultiplyRound( + Vector128 input0, + int weight0, + Vector128 input1, + int weight1, + int cosBit, + Vector128 rounding) + { + // Widening preserves lane order on every Vector128 implementation. The explicit clamp gives Narrow the + // signed-saturating demotion semantics used by Highway on x86, Arm, and WebAssembly. + (Vector128 input0Lower, Vector128 input0Upper) = Vector128.Widen(input0); + (Vector128 input1Lower, Vector128 input1Upper) = Vector128.Widen(input1); + + Vector128 weight0Vector = Vector128.Create(weight0); + Vector128 weight1Vector = Vector128.Create(weight1); + Vector128 lower = ((input0Lower * weight0Vector) + (input1Lower * weight1Vector) + rounding) >> cosBit; + Vector128 upper = ((input0Upper * weight0Vector) + (input1Upper * weight1Vector) + rounding) >> cosBit; + Vector128 minimum = Vector128.Create((int)short.MinValue); + Vector128 maximum = Vector128.Create((int)short.MaxValue); + + lower = Vector128.Clamp(lower, minimum, maximum); + upper = Vector128.Clamp(upper, minimum, maximum); + return Vector128.Narrow(lower, upper); + } + + /// + /// Calculates and narrows two weighted 256-bit signed sixteen-bit vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MultiplyRound( + Vector256 input0, + int weight0, + Vector256 input1, + int weight1, + int cosBit, + Vector256 rounding) + { + // The AVX2 path mirrors Highway's WidenMulPairwiseAdd primitive: adjacent Int16 products become Int32 + // sums, then VPACKSSDW restores the original lane width with signed saturation. + Vector256 lowerInputs = Avx2.UnpackLow(input0, input1); + Vector256 upperInputs = Avx2.UnpackHigh(input0, input1); + Vector256 weights = Avx2.UnpackLow(Vector256.Create((short)weight0), Vector256.Create((short)weight1)); + Vector256 lower = (Avx2.MultiplyAddAdjacent(lowerInputs, weights) + rounding) >> cosBit; + Vector256 upper = (Avx2.MultiplyAddAdjacent(upperInputs, weights) + rounding) >> cosBit; + return Avx2.PackSignedSaturate(lower, upper); + } + + /// + /// Calculates and narrows two weighted 512-bit signed sixteen-bit vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 MultiplyRound( + Vector512 input0, + int weight0, + Vector512 input1, + int weight1, + int cosBit, + Vector512 rounding) + { + Vector512 lowerInputs = Avx512BW.UnpackLow(input0, input1); + Vector512 upperInputs = Avx512BW.UnpackHigh(input0, input1); + Vector512 weights = Avx512BW.UnpackLow(Vector512.Create((short)weight0), Vector512.Create((short)weight1)); + Vector512 lower = (Avx512BW.MultiplyAddAdjacent(lowerInputs, weights) + rounding) >> cosBit; + Vector512 upper = (Avx512BW.MultiplyAddAdjacent(upperInputs, weights) + rounding) >> cosBit; + return Avx512BW.PackSignedSaturate(lower, upper); + } + + /// + /// Calculates and narrows four weighted 128-bit signed sixteen-bit vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MultiplyRound( + Vector128 input0, + int weight0, + Vector128 input1, + int weight1, + Vector128 input2, + int weight2, + Vector128 input3, + int weight3, + int cosBit, + Vector128 rounding) + { + (Vector128 input0Lower, Vector128 input0Upper) = Vector128.Widen(input0); + (Vector128 input1Lower, Vector128 input1Upper) = Vector128.Widen(input1); + (Vector128 input2Lower, Vector128 input2Upper) = Vector128.Widen(input2); + (Vector128 input3Lower, Vector128 input3Upper) = Vector128.Widen(input3); + + Vector128 lower = ((input0Lower * Vector128.Create(weight0)) + + (input1Lower * Vector128.Create(weight1)) + + (input2Lower * Vector128.Create(weight2)) + + (input3Lower * Vector128.Create(weight3)) + + rounding) >> cosBit; + + Vector128 upper = ((input0Upper * Vector128.Create(weight0)) + + (input1Upper * Vector128.Create(weight1)) + + (input2Upper * Vector128.Create(weight2)) + + (input3Upper * Vector128.Create(weight3)) + + rounding) >> cosBit; + + Vector128 minimum = Vector128.Create((int)short.MinValue); + Vector128 maximum = Vector128.Create((int)short.MaxValue); + lower = Vector128.Clamp(lower, minimum, maximum); + upper = Vector128.Clamp(upper, minimum, maximum); + return Vector128.Narrow(lower, upper); + } + + /// + /// Calculates and narrows four weighted 256-bit signed sixteen-bit vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 MultiplyRound( + Vector256 input0, + int weight0, + Vector256 input1, + int weight1, + Vector256 input2, + int weight2, + Vector256 input3, + int weight3, + int cosBit, + Vector256 rounding) + { + Vector256 weights01 = Avx2.UnpackLow(Vector256.Create((short)weight0), Vector256.Create((short)weight1)); + Vector256 weights23 = Avx2.UnpackLow(Vector256.Create((short)weight2), Vector256.Create((short)weight3)); + Vector256 lower = Avx2.MultiplyAddAdjacent(Avx2.UnpackLow(input0, input1), weights01) + + Avx2.MultiplyAddAdjacent(Avx2.UnpackLow(input2, input3), weights23); + + Vector256 upper = Avx2.MultiplyAddAdjacent(Avx2.UnpackHigh(input0, input1), weights01) + + Avx2.MultiplyAddAdjacent(Avx2.UnpackHigh(input2, input3), weights23); + + lower = (lower + rounding) >> cosBit; + upper = (upper + rounding) >> cosBit; + return Avx2.PackSignedSaturate(lower, upper); + } + + /// + /// Calculates and narrows four weighted 512-bit signed sixteen-bit vectors. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 MultiplyRound( + Vector512 input0, + int weight0, + Vector512 input1, + int weight1, + Vector512 input2, + int weight2, + Vector512 input3, + int weight3, + int cosBit, + Vector512 rounding) + { + Vector512 weights01 = Avx512BW.UnpackLow(Vector512.Create((short)weight0), Vector512.Create((short)weight1)); + Vector512 weights23 = Avx512BW.UnpackLow(Vector512.Create((short)weight2), Vector512.Create((short)weight3)); + Vector512 lower = Avx512BW.MultiplyAddAdjacent(Avx512BW.UnpackLow(input0, input1), weights01) + + Avx512BW.MultiplyAddAdjacent(Avx512BW.UnpackLow(input2, input3), weights23); + + Vector512 upper = Avx512BW.MultiplyAddAdjacent(Avx512BW.UnpackHigh(input0, input1), weights01) + + Avx512BW.MultiplyAddAdjacent(Avx512BW.UnpackHigh(input2, input3), weights23); + + lower = (lower + rounding) >> cosBit; + upper = (upper + rounding) >> cosBit; + return Avx512BW.PackSignedSaturate(lower, upper); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs new file mode 100644 index 000000000..c194fcbf7 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs @@ -0,0 +1,311 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the forward asymmetric discrete sine transform stage networks. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the four-point forward asymmetric discrete sine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The unused transform-stage buffer. + /// The fixed-point precision of the sine constants. + public static void Adst4( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + _ = step; + + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + TValue input0 = input[0]; + TValue input1 = input[1]; + TValue input2 = input[2]; + TValue input3 = input[3]; + TValue input01 = Av1ForwardTransformArithmetic.Add(input0, input1); + + // Highway forms x0 + x1 in the native lane width before widening the products. Retaining that intermediate + // is observable for Int16 overflow and is therefore part of the reference stage network, not an algebraic + // simplification opportunity. + output[0] = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); + output[1] = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); + output[2] = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); + + // The final output is Highway's widened w2 - w0 + 3 * v5 sequence expressed with the same unrounded + // products. All four outputs then share the single normative fixed-point rounding point. + output[3] = Av1ForwardTransformArithmetic.MultiplyAddRound( + sinpi[4] - sinpi[1], + input0, + -sinpi[1] - sinpi[2], + input1, + sinpi[3], + input2, + sinpi[2] - sinpi[4], + input3, + cosBit, + in rounding); + } + + /// + /// Applies the eight-point forward asymmetric discrete sine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values and first transform-stage buffer. + /// The second transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Adst8( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 applies the ADST input permutation and signs. The following stages can then use the same adjacent + // butterfly layout across every scalar and SIMD instantiation. + output[0] = input[0]; + output[1] = Av1ForwardTransformArithmetic.Negate(input[7]); + output[2] = Av1ForwardTransformArithmetic.Negate(input[3]); + output[3] = input[4]; + output[4] = Av1ForwardTransformArithmetic.Negate(input[1]); + output[5] = input[6]; + output[6] = input[2]; + output[7] = Av1ForwardTransformArithmetic.Negate(input[5]); + + // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. + step[0] = output[0]; + step[1] = output[1]; + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], cospi[32], output[2], output[3], out step[2], out step[3], cosBit, in rounding); + step[4] = output[4]; + step[5] = output[5]; + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], cospi[32], output[6], output[7], out step[6], out step[7], cosBit, in rounding); + + // Stage 3 combines the rotated and copied pairs into two independent four-value groups. + for (int group = 0; group < 8; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + step[group + i], + step[group + i + 2], + out output[group + i], + out output[group + i + 2]); + } + } + + // Stage 4 rotates the upper group by pi/8 and retains the completed lower group. + for (int i = 0; i < 4; i++) + { + step[i] = output[i]; + } + + step[4] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], output[4], cospi[48], output[5], cosBit, in rounding); + step[5] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[48], output[4], -cospi[16], output[5], cosBit, in rounding); + step[6] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[48], output[6], cospi[16], output[7], cosBit, in rounding); + step[7] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], output[6], cospi[48], output[7], cosBit, in rounding); + + // Stage 5 creates the four final butterfly pairs spanning the two groups. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(step[i], step[i + 4], out output[i], out output[i + 4]); + } + + // Stage 6 applies the remaining odd angles. Each result is placed in step for the fixed ADST permutation. + step[0] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[4], output[0], cospi[60], output[1], cosBit, in rounding); + step[1] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[60], output[0], -cospi[4], output[1], cosBit, in rounding); + step[2] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[20], output[2], cospi[44], output[3], cosBit, in rounding); + step[3] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[44], output[2], -cospi[20], output[3], cosBit, in rounding); + step[4] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[36], output[4], cospi[28], output[5], cosBit, in rounding); + step[5] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[28], output[4], -cospi[36], output[5], cosBit, in rounding); + step[6] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[52], output[6], cospi[12], output[7], cosBit, in rounding); + step[7] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[12], output[6], -cospi[52], output[7], cosBit, in rounding); + + // Stage 7 is the normative ADST output permutation. + output[0] = step[1]; + output[1] = step[6]; + output[2] = step[3]; + output[3] = step[4]; + output[4] = step[5]; + output[5] = step[2]; + output[6] = step[7]; + output[7] = step[0]; + } + + /// + /// Applies the sixteen-point forward asymmetric discrete sine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values and first transform-stage buffer. + /// The second transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Adst16( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 applies the bit-reversed ADST input order and its alternating signs. + output[0] = input[0]; + output[1] = Av1ForwardTransformArithmetic.Negate(input[15]); + output[2] = Av1ForwardTransformArithmetic.Negate(input[7]); + output[3] = input[8]; + output[4] = Av1ForwardTransformArithmetic.Negate(input[3]); + output[5] = input[12]; + output[6] = input[4]; + output[7] = Av1ForwardTransformArithmetic.Negate(input[11]); + output[8] = Av1ForwardTransformArithmetic.Negate(input[1]); + output[9] = input[14]; + output[10] = input[6]; + output[11] = Av1ForwardTransformArithmetic.Negate(input[9]); + output[12] = input[2]; + output[13] = Av1ForwardTransformArithmetic.Negate(input[13]); + output[14] = Av1ForwardTransformArithmetic.Negate(input[5]); + output[15] = input[10]; + + // Stage 2 rotates the second pair in each group of four and copies the first pair unchanged. + for (int group = 0; group < 16; group += 4) + { + step[group] = output[group]; + step[group + 1] = output[group + 1]; + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + output[group + 2], + output[group + 3], + out step[group + 2], + out step[group + 3], + cosBit, + in rounding); + } + + // Stage 3 combines adjacent pairs within each group of four. + for (int group = 0; group < 16; group += 4) + { + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + step[group + i], + step[group + i + 2], + out output[group + i], + out output[group + i + 2]); + } + } + + // Stage 4 rotates the upper pair of each eight-value group by pi/8. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + step[group + i] = output[group + i]; + } + + step[group + 4] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], output[group + 4], cospi[48], output[group + 5], cosBit, in rounding); + step[group + 5] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[48], output[group + 4], -cospi[16], output[group + 5], cosBit, in rounding); + step[group + 6] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[48], output[group + 6], cospi[16], output[group + 7], cosBit, in rounding); + step[group + 7] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], output[group + 6], cospi[48], output[group + 7], cosBit, in rounding); + } + + // Stage 5 combines the lower and upper quartets within each eight-value group. + for (int group = 0; group < 16; group += 8) + { + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + step[group + i], + step[group + i + 4], + out output[group + i], + out output[group + i + 4]); + } + } + + // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. + for (int i = 0; i < 8; i++) + { + step[i] = output[i]; + } + + step[8] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[8], output[8], cospi[56], output[9], cosBit, in rounding); + step[9] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[56], output[8], -cospi[8], output[9], cosBit, in rounding); + step[10] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[40], output[10], cospi[24], output[11], cosBit, in rounding); + step[11] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[24], output[10], -cospi[40], output[11], cosBit, in rounding); + step[12] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[56], output[12], cospi[8], output[13], cosBit, in rounding); + step[13] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[8], output[12], cospi[56], output[13], cosBit, in rounding); + step[14] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[24], output[14], cospi[40], output[15], cosBit, in rounding); + step[15] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[40], output[14], cospi[24], output[15], cosBit, in rounding); + + // Stage 7 creates the eight final butterfly pairs spanning both octets. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(step[i], step[i + 8], out output[i], out output[i + 8]); + } + + // Stage 8 applies the final odd-angle rotations before the fixed output permutation. + ReadOnlySpan firstWeights = [2, 10, 18, 26, 34, 42, 50, 58]; + + for (int pair = 0; pair < 8; pair++) + { + int first = firstWeights[pair]; + int second = 64 - first; + int index = pair * 2; + + step[index] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[first], output[index], cospi[second], output[index + 1], cosBit, in rounding); + step[index + 1] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[second], output[index], -cospi[first], output[index + 1], cosBit, in rounding); + } + + // Stage 9 maps the rotated input to ascending AV1 ADST coefficient order. + ReadOnlySpan permutation = [1, 14, 3, 12, 5, 10, 7, 8, 9, 6, 11, 4, 13, 2, 15, 0]; + + for (int i = 0; i < 16; i++) + { + output[i] = step[permutation[i]]; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs new file mode 100644 index 000000000..9126069d0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs @@ -0,0 +1,120 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the sixteen-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the sixteen-point forward discrete cosine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Dct16( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + Av1ForwardTransformArithmetic.AddSubtract(input[0], input[15], out output[0], out output[15]); + Av1ForwardTransformArithmetic.AddSubtract(input[1], input[14], out output[1], out output[14]); + Av1ForwardTransformArithmetic.AddSubtract(input[2], input[13], out output[2], out output[13]); + Av1ForwardTransformArithmetic.AddSubtract(input[3], input[12], out output[3], out output[12]); + Av1ForwardTransformArithmetic.AddSubtract(input[4], input[11], out output[4], out output[11]); + Av1ForwardTransformArithmetic.AddSubtract(input[5], input[10], out output[5], out output[10]); + Av1ForwardTransformArithmetic.AddSubtract(input[6], input[9], out output[6], out output[9]); + Av1ForwardTransformArithmetic.AddSubtract(input[7], input[8], out output[7], out output[8]); + + // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + Av1ForwardTransformArithmetic.AddSubtract(output[0], output[7], out step[0], out step[7]); + Av1ForwardTransformArithmetic.AddSubtract(output[1], output[6], out step[1], out step[6]); + Av1ForwardTransformArithmetic.AddSubtract(output[2], output[5], out step[2], out step[5]); + Av1ForwardTransformArithmetic.AddSubtract(output[3], output[4], out step[3], out step[4]); + + step[8] = output[8]; + step[9] = output[9]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[10], output[13], out step[10], out step[13], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[11], output[12], out step[11], out step[12], cosBit, in rounding); + step[14] = output[14]; + step[15] = output[15]; + + // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. + Av1ForwardTransformArithmetic.AddSubtract(step[0], step[3], out output[0], out output[3]); + Av1ForwardTransformArithmetic.AddSubtract(step[1], step[2], out output[1], out output[2]); + + output[4] = step[4]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[5], step[6], out output[5], out output[6], cosBit, in rounding); + output[7] = step[7]; + Av1ForwardTransformArithmetic.AddSubtract(step[8], step[11], out output[8], out output[11]); + Av1ForwardTransformArithmetic.AddSubtract(step[9], step[10], out output[9], out output[10]); + Av1ForwardTransformArithmetic.AddSubtract(step[14], step[13], out output[14], out output[13]); + Av1ForwardTransformArithmetic.AddSubtract(step[15], step[12], out output[15], out output[12]); + + // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. + Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], output[0], output[1], out step[0], out step[1], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], output[3], output[2], out step[2], out step[3], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(output[4], output[5], out step[4], out step[5]); + Av1ForwardTransformArithmetic.AddSubtract(output[7], output[6], out step[7], out step[6]); + + step[8] = output[8]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[9], output[14], out step[9], out step[14], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[10], output[13], out step[10], out step[13], cosBit, in rounding); + step[11] = output[11]; + step[12] = output[12]; + step[15] = output[15]; + + // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], step[7], step[4], out output[4], out output[7], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], step[6], step[5], out output[5], out output[6], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(step[8], step[9], out output[8], out output[9]); + Av1ForwardTransformArithmetic.AddSubtract(step[11], step[10], out output[11], out output[10]); + Av1ForwardTransformArithmetic.AddSubtract(step[12], step[13], out output[12], out output[13]); + Av1ForwardTransformArithmetic.AddSubtract(step[15], step[14], out output[15], out output[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + Av1ForwardTransformArithmetic.Butterfly(cospi[4], cospi[60], output[15], output[8], out step[8], out step[15], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[36], cospi[28], output[14], output[9], out step[9], out step[14], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[20], cospi[44], output[13], output[10], out step[10], out step[13], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[52], cospi[12], output[12], output[11], out step[11], out step[12], cosBit, in rounding); + + // Stage 7 permutes the staged values into ascending AV1 coefficient order. + output[0] = step[0]; + output[1] = step[8]; + output[2] = step[4]; + output[3] = step[12]; + output[4] = step[2]; + output[5] = step[10]; + output[6] = step[6]; + output[7] = step[14]; + output[8] = step[1]; + output[9] = step[9]; + output[10] = step[5]; + output[11] = step[13]; + output[12] = step[3]; + output[13] = step[11]; + output[14] = step[7]; + output[15] = step[15]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs new file mode 100644 index 000000000..9d3d47d20 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs @@ -0,0 +1,224 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the thirty-two-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the thirty-two-point forward discrete cosine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Dct32( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + for (int index = 0; index < 16; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(input[index], input[31 - index], out output[index], out output[31 - index]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + for (int index = 0; index < 8; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(output[index], output[15 - index], out step[index], out step[15 - index]); + } + + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[20], output[27], out step[20], out step[27], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[21], output[26], out step[21], out step[26], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[22], output[25], out step[22], out step[25], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[23], output[24], out step[23], out step[24], cosBit, in rounding); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(step[index], step[7 - index], out output[index], out output[7 - index]); + } + + output[8] = step[8]; + output[9] = step[9]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[10], step[13], out output[10], out output[13], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[11], step[12], out output[11], out output[12], cosBit, in rounding); + output[14] = step[14]; + output[15] = step[15]; + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(step[16 + index], step[23 - index], out output[16 + index], out output[23 - index]); + Av1ForwardTransformArithmetic.AddSubtract(step[31 - index], step[24 + index], out output[31 - index], out output[24 + index]); + } + + // Stage 4 continues the factorization as independent eight-sample groups. + Av1ForwardTransformArithmetic.AddSubtract(output[0], output[3], out step[0], out step[3]); + Av1ForwardTransformArithmetic.AddSubtract(output[1], output[2], out step[1], out step[2]); + + step[4] = output[4]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[5], output[6], out step[5], out step[6], cosBit, in rounding); + step[7] = output[7]; + Av1ForwardTransformArithmetic.AddSubtract(output[8], output[11], out step[8], out step[11]); + Av1ForwardTransformArithmetic.AddSubtract(output[9], output[10], out step[9], out step[10]); + Av1ForwardTransformArithmetic.AddSubtract(output[14], output[13], out step[14], out step[13]); + Av1ForwardTransformArithmetic.AddSubtract(output[15], output[12], out step[15], out step[12]); + + step[16] = output[16]; + step[17] = output[17]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[18], output[29], out step[18], out step[29], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[19], output[28], out step[19], out step[28], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[20], output[27], out step[20], out step[27], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[21], output[26], out step[21], out step[26], cosBit, in rounding); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. + Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], step[0], step[1], out output[0], out output[1], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], step[3], step[2], out output[2], out output[3], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(step[4], step[5], out output[4], out output[5]); + Av1ForwardTransformArithmetic.AddSubtract(step[7], step[6], out output[7], out output[6]); + + output[8] = step[8]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], step[9], step[14], out output[9], out output[14], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], step[10], step[13], out output[10], out output[13], cosBit, in rounding); + output[11] = step[11]; + output[12] = step[12]; + output[15] = step[15]; + Av1ForwardTransformArithmetic.AddSubtract(step[16], step[19], out output[16], out output[19]); + Av1ForwardTransformArithmetic.AddSubtract(step[17], step[18], out output[17], out output[18]); + Av1ForwardTransformArithmetic.AddSubtract(step[23], step[20], out output[23], out output[20]); + Av1ForwardTransformArithmetic.AddSubtract(step[22], step[21], out output[22], out output[21]); + Av1ForwardTransformArithmetic.AddSubtract(step[24], step[27], out output[24], out output[27]); + Av1ForwardTransformArithmetic.AddSubtract(step[25], step[26], out output[25], out output[26]); + Av1ForwardTransformArithmetic.AddSubtract(step[31], step[28], out output[31], out output[28]); + Av1ForwardTransformArithmetic.AddSubtract(step[30], step[29], out output[30], out output[29]); + + // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], output[7], output[4], out step[4], out step[7], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], output[6], output[5], out step[5], out step[6], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(output[8], output[9], out step[8], out step[9]); + Av1ForwardTransformArithmetic.AddSubtract(output[11], output[10], out step[11], out step[10]); + Av1ForwardTransformArithmetic.AddSubtract(output[12], output[13], out step[12], out step[13]); + Av1ForwardTransformArithmetic.AddSubtract(output[15], output[14], out step[15], out step[14]); + + step[16] = output[16]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], output[17], output[30], out step[17], out step[30], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], output[18], output[29], out step[18], out step[29], cosBit, in rounding); + step[19] = output[19]; + step[20] = output[20]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], output[21], output[26], out step[21], out step[26], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], output[22], output[25], out step[22], out step[25], cosBit, in rounding); + step[23] = output[23]; + step[24] = output[24]; + step[27] = output[27]; + step[28] = output[28]; + step[31] = output[31]; + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + Av1ForwardTransformArithmetic.Butterfly(cospi[4], cospi[60], step[15], step[8], out output[8], out output[15], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[36], cospi[28], step[14], step[9], out output[9], out output[14], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[20], cospi[44], step[13], step[10], out output[10], out output[13], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[52], cospi[12], step[12], step[11], out output[11], out output[12], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(step[16], step[17], out output[16], out output[17]); + Av1ForwardTransformArithmetic.AddSubtract(step[19], step[18], out output[19], out output[18]); + Av1ForwardTransformArithmetic.AddSubtract(step[20], step[21], out output[20], out output[21]); + Av1ForwardTransformArithmetic.AddSubtract(step[23], step[22], out output[23], out output[22]); + Av1ForwardTransformArithmetic.AddSubtract(step[24], step[25], out output[24], out output[25]); + Av1ForwardTransformArithmetic.AddSubtract(step[27], step[26], out output[27], out output[26]); + Av1ForwardTransformArithmetic.AddSubtract(step[28], step[29], out output[28], out output[29]); + Av1ForwardTransformArithmetic.AddSubtract(step[31], step[30], out output[31], out output[30]); + + // Stage 8 merges the final odd-frequency pairs before their terminal rotations. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + Av1ForwardTransformArithmetic.Butterfly(cospi[2], cospi[62], output[31], output[16], out step[16], out step[31], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[34], cospi[30], output[30], output[17], out step[17], out step[30], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[18], cospi[46], output[29], output[18], out step[18], out step[29], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[50], cospi[14], output[28], output[19], out step[19], out step[28], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[10], cospi[54], output[27], output[20], out step[20], out step[27], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[42], cospi[22], output[26], output[21], out step[21], out step[26], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[26], cospi[38], output[25], output[22], out step[22], out step[25], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[58], cospi[6], output[24], output[23], out step[23], out step[24], cosBit, in rounding); + + // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. + output[0] = step[0]; + output[1] = step[16]; + output[2] = step[8]; + output[3] = step[24]; + output[4] = step[4]; + output[5] = step[20]; + output[6] = step[12]; + output[7] = step[28]; + output[8] = step[2]; + output[9] = step[18]; + output[10] = step[10]; + output[11] = step[26]; + output[12] = step[6]; + output[13] = step[22]; + output[14] = step[14]; + output[15] = step[30]; + output[16] = step[1]; + output[17] = step[17]; + output[18] = step[9]; + output[19] = step[25]; + output[20] = step[5]; + output[21] = step[21]; + output[22] = step[13]; + output[23] = step[29]; + output[24] = step[3]; + output[25] = step[19]; + output[26] = step[11]; + output[27] = step[27]; + output[28] = step[7]; + output[29] = step[23]; + output[30] = step[15]; + output[31] = step[31]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs new file mode 100644 index 000000000..824fbd612 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs @@ -0,0 +1,41 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the four-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the four-point forward discrete cosine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Dct4( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + // Libaom forms both outputs of each mirror pair together. This preserves the saturating Int16 AddSub + // primitive used by Highway while the Int32 and scalar specializations retain their native arithmetic. + Av1ForwardTransformArithmetic.AddSubtract(input[0], input[3], out output[0], out output[3]); + Av1ForwardTransformArithmetic.AddSubtract(input[1], input[2], out output[1], out output[2]); + + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], output[0], output[1], out step[0], out step[2], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], output[3], output[2], out step[1], out step[3], cosBit, in rounding); + + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs new file mode 100644 index 000000000..df9175d5e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs @@ -0,0 +1,437 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the sixty-four-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the sixty-four-point forward discrete cosine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Dct64( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + for (int index = 0; index < 32; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(input[index], input[63 - index], out output[index], out output[63 - index]); + } + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + for (int index = 0; index < 16; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(output[index], output[31 - index], out step[index], out step[31 - index]); + } + + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = output[36]; + step[37] = output[37]; + step[38] = output[38]; + step[39] = output[39]; + for (int index = 0; index < 8; index++) + { + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], cospi[32], output[40 + index], output[55 - index], out step[40 + index], out step[55 - index], cosBit, in rounding); + } + + step[56] = output[56]; + step[57] = output[57]; + step[58] = output[58]; + step[59] = output[59]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. + for (int index = 0; index < 8; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(step[index], step[15 - index], out output[index], out output[15 - index]); + } + + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], cospi[32], step[20 + index], step[27 - index], out output[20 + index], out output[27 - index], cosBit, in rounding); + } + + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + for (int index = 0; index < 8; index++) + { + Av1ForwardTransformArithmetic.AddSubtract( + step[32 + index], step[47 - index], out output[32 + index], out output[47 - index]); + Av1ForwardTransformArithmetic.AddSubtract( + step[63 - index], step[48 + index], out output[63 - index], out output[48 + index]); + } + + // Stage 4 continues the factorization as independent sixteen-sample groups. + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.AddSubtract(output[index], output[7 - index], out step[index], out step[7 - index]); + } + + step[8] = output[8]; + step[9] = output[9]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[10], output[13], out step[10], out step[13], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[11], output[12], out step[11], out step[12], cosBit, in rounding); + step[14] = output[14]; + step[15] = output[15]; + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.AddSubtract( + output[16 + index], output[23 - index], out step[16 + index], out step[23 - index]); + Av1ForwardTransformArithmetic.AddSubtract( + output[31 - index], output[24 + index], out step[31 - index], out step[24 + index]); + } + + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], cospi[48], output[36 + index], output[59 - index], out step[36 + index], out step[59 - index], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], -cospi[16], output[40 + index], output[55 - index], out step[40 + index], out step[55 - index], cosBit, in rounding); + } + + step[44] = output[44]; + step[45] = output[45]; + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = output[50]; + step[51] = output[51]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. + Av1ForwardTransformArithmetic.AddSubtract(step[0], step[3], out output[0], out output[3]); + Av1ForwardTransformArithmetic.AddSubtract(step[1], step[2], out output[1], out output[2]); + + output[4] = step[4]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[5], step[6], out output[5], out output[6], cosBit, in rounding); + output[7] = step[7]; + Av1ForwardTransformArithmetic.AddSubtract(step[8], step[11], out output[8], out output[11]); + Av1ForwardTransformArithmetic.AddSubtract(step[9], step[10], out output[9], out output[10]); + Av1ForwardTransformArithmetic.AddSubtract(step[14], step[13], out output[14], out output[13]); + Av1ForwardTransformArithmetic.AddSubtract(step[15], step[12], out output[15], out output[12]); + output[16] = step[16]; + output[17] = step[17]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], step[18], step[29], out output[18], out output[29], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], step[19], step[28], out output[19], out output[28], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], step[20], step[27], out output[20], out output[27], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], step[21], step[26], out output[21], out output[26], cosBit, in rounding); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[30] = step[30]; + output[31] = step[31]; + for (int index = 0; index < 4; index++) + { + Av1ForwardTransformArithmetic.AddSubtract( + step[32 + index], step[39 - index], out output[32 + index], out output[39 - index]); + Av1ForwardTransformArithmetic.AddSubtract( + step[47 - index], step[40 + index], out output[47 - index], out output[40 + index]); + Av1ForwardTransformArithmetic.AddSubtract( + step[48 + index], step[55 - index], out output[48 + index], out output[55 - index]); + Av1ForwardTransformArithmetic.AddSubtract( + step[63 - index], step[56 + index], out output[63 - index], out output[56 + index]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. + Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], output[0], output[1], out step[0], out step[1], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], output[3], output[2], out step[2], out step[3], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(output[4], output[5], out step[4], out step[5]); + Av1ForwardTransformArithmetic.AddSubtract(output[7], output[6], out step[7], out step[6]); + step[8] = output[8]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[9], output[14], out step[9], out step[14], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[10], output[13], out step[10], out step[13], cosBit, in rounding); + step[11] = output[11]; + step[12] = output[12]; + step[15] = output[15]; + Av1ForwardTransformArithmetic.AddSubtract(output[16], output[19], out step[16], out step[19]); + Av1ForwardTransformArithmetic.AddSubtract(output[17], output[18], out step[17], out step[18]); + Av1ForwardTransformArithmetic.AddSubtract(output[23], output[20], out step[23], out step[20]); + Av1ForwardTransformArithmetic.AddSubtract(output[22], output[21], out step[22], out step[21]); + Av1ForwardTransformArithmetic.AddSubtract(output[24], output[27], out step[24], out step[27]); + Av1ForwardTransformArithmetic.AddSubtract(output[25], output[26], out step[25], out step[26]); + Av1ForwardTransformArithmetic.AddSubtract(output[31], output[28], out step[31], out step[28]); + Av1ForwardTransformArithmetic.AddSubtract(output[30], output[29], out step[30], out step[29]); + step[32] = output[32]; + step[33] = output[33]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], output[34], output[61], out step[34], out step[61], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], output[35], output[60], out step[35], out step[60], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], output[36], output[59], out step[36], out step[59], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], output[37], output[58], out step[37], out step[58], cosBit, in rounding); + step[38] = output[38]; + step[39] = output[39]; + step[40] = output[40]; + step[41] = output[41]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], output[42], output[53], out step[42], out step[53], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], output[43], output[52], out step[43], out step[52], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], output[44], output[51], out step[44], out step[51], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], output[45], output[50], out step[45], out step[50], cosBit, in rounding); + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[54] = output[54]; + step[55] = output[55]; + step[56] = output[56]; + step[57] = output[57]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], step[7], step[4], out output[4], out output[7], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], step[6], step[5], out output[5], out output[6], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(step[8], step[9], out output[8], out output[9]); + Av1ForwardTransformArithmetic.AddSubtract(step[11], step[10], out output[11], out output[10]); + Av1ForwardTransformArithmetic.AddSubtract(step[12], step[13], out output[12], out output[13]); + Av1ForwardTransformArithmetic.AddSubtract(step[15], step[14], out output[15], out output[14]); + output[16] = step[16]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], step[17], step[30], out output[17], out output[30], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], step[18], step[29], out output[18], out output[29], cosBit, in rounding); + output[19] = step[19]; + output[20] = step[20]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], step[21], step[26], out output[21], out output[26], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], step[22], step[25], out output[22], out output[25], cosBit, in rounding); + output[23] = step[23]; + output[24] = step[24]; + output[27] = step[27]; + output[28] = step[28]; + output[31] = step[31]; + for (int offset = 32; offset < 64; offset += 8) + { + Av1ForwardTransformArithmetic.AddSubtract(step[offset], step[offset + 3], out output[offset], out output[offset + 3]); + Av1ForwardTransformArithmetic.AddSubtract(step[offset + 1], step[offset + 2], out output[offset + 1], out output[offset + 2]); + Av1ForwardTransformArithmetic.AddSubtract(step[offset + 7], step[offset + 4], out output[offset + 7], out output[offset + 4]); + Av1ForwardTransformArithmetic.AddSubtract(step[offset + 6], step[offset + 5], out output[offset + 6], out output[offset + 5]); + } + + // Stage 8 applies the next level of odd-frequency rotations. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + Av1ForwardTransformArithmetic.Butterfly(cospi[4], cospi[60], output[15], output[8], out step[8], out step[15], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[36], cospi[28], output[14], output[9], out step[9], out step[14], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[20], cospi[44], output[13], output[10], out step[10], out step[13], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[52], cospi[12], output[12], output[11], out step[11], out step[12], cosBit, in rounding); + for (int offset = 16; offset < 32; offset += 4) + { + Av1ForwardTransformArithmetic.AddSubtract(output[offset], output[offset + 1], out step[offset], out step[offset + 1]); + Av1ForwardTransformArithmetic.AddSubtract(output[offset + 3], output[offset + 2], out step[offset + 3], out step[offset + 2]); + } + + step[32] = output[32]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[4], cospi[60], output[33], output[62], out step[33], out step[62], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[60], -cospi[4], output[34], output[61], out step[34], out step[61], cosBit, in rounding); + step[35] = output[35]; + step[36] = output[36]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[36], cospi[28], output[37], output[58], out step[37], out step[58], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[28], -cospi[36], output[38], output[57], out step[38], out step[57], cosBit, in rounding); + step[39] = output[39]; + step[40] = output[40]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[20], cospi[44], output[41], output[54], out step[41], out step[54], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[44], -cospi[20], output[42], output[53], out step[42], out step[53], cosBit, in rounding); + step[43] = output[43]; + step[44] = output[44]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[52], cospi[12], output[45], output[50], out step[45], out step[50], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(-cospi[12], -cospi[52], output[46], output[49], out step[46], out step[49], cosBit, in rounding); + step[47] = output[47]; + step[48] = output[48]; + step[51] = output[51]; + step[52] = output[52]; + step[55] = output[55]; + step[56] = output[56]; + step[59] = output[59]; + step[60] = output[60]; + step[63] = output[63]; + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = step[10]; + output[11] = step[11]; + output[12] = step[12]; + output[13] = step[13]; + output[14] = step[14]; + output[15] = step[15]; + Av1ForwardTransformArithmetic.Butterfly(cospi[2], cospi[62], step[31], step[16], out output[16], out output[31], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[34], cospi[30], step[30], step[17], out output[17], out output[30], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[18], cospi[46], step[29], step[18], out output[18], out output[29], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[50], cospi[14], step[28], step[19], out output[19], out output[28], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[10], cospi[54], step[27], step[20], out output[20], out output[27], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[42], cospi[22], step[26], step[21], out output[21], out output[26], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[26], cospi[38], step[25], step[22], out output[22], out output[25], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[58], cospi[6], step[24], step[23], out output[23], out output[24], cosBit, in rounding); + for (int offset = 32; offset < 64; offset += 4) + { + Av1ForwardTransformArithmetic.AddSubtract(step[offset], step[offset + 1], out output[offset], out output[offset + 1]); + Av1ForwardTransformArithmetic.AddSubtract(step[offset + 3], step[offset + 2], out output[offset + 3], out output[offset + 2]); + } + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = output[20]; + step[21] = output[21]; + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = output[26]; + step[27] = output[27]; + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + Av1ForwardTransformArithmetic.Butterfly(cospi[1], cospi[63], output[63], output[32], out step[32], out step[63], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[33], cospi[31], output[62], output[33], out step[33], out step[62], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[17], cospi[47], output[61], output[34], out step[34], out step[61], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[49], cospi[15], output[60], output[35], out step[35], out step[60], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[9], cospi[55], output[59], output[36], out step[36], out step[59], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[41], cospi[23], output[58], output[37], out step[37], out step[58], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[25], cospi[39], output[57], output[38], out step[38], out step[57], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[57], cospi[7], output[56], output[39], out step[39], out step[56], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[5], cospi[59], output[55], output[40], out step[40], out step[55], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[37], cospi[27], output[54], output[41], out step[41], out step[54], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[21], cospi[43], output[53], output[42], out step[42], out step[53], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[53], cospi[11], output[52], output[43], out step[43], out step[52], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[13], cospi[51], output[51], output[44], out step[44], out step[51], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[45], cospi[19], output[50], output[45], out step[45], out step[50], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[29], cospi[35], output[49], output[46], out step[46], out step[49], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[61], cospi[3], output[48], output[47], out step[47], out step[48], cosBit, in rounding); + + // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. + output[0] = step[0]; + output[1] = step[32]; + output[2] = step[16]; + output[3] = step[48]; + output[4] = step[8]; + output[5] = step[40]; + output[6] = step[24]; + output[7] = step[56]; + output[8] = step[4]; + output[9] = step[36]; + output[10] = step[20]; + output[11] = step[52]; + output[12] = step[12]; + output[13] = step[44]; + output[14] = step[28]; + output[15] = step[60]; + output[16] = step[2]; + output[17] = step[34]; + output[18] = step[18]; + output[19] = step[50]; + output[20] = step[10]; + output[21] = step[42]; + output[22] = step[26]; + output[23] = step[58]; + output[24] = step[6]; + output[25] = step[38]; + output[26] = step[22]; + output[27] = step[54]; + output[28] = step[14]; + output[29] = step[46]; + output[30] = step[30]; + output[31] = step[62]; + output[32] = step[1]; + output[33] = step[33]; + output[34] = step[17]; + output[35] = step[49]; + output[36] = step[9]; + output[37] = step[41]; + output[38] = step[25]; + output[39] = step[57]; + output[40] = step[5]; + output[41] = step[37]; + output[42] = step[21]; + output[43] = step[53]; + output[44] = step[13]; + output[45] = step[45]; + output[46] = step[29]; + output[47] = step[61]; + output[48] = step[3]; + output[49] = step[35]; + output[50] = step[19]; + output[51] = step[51]; + output[52] = step[11]; + output[53] = step[43]; + output[54] = step[27]; + output[55] = step[59]; + output[56] = step[7]; + output[57] = step[39]; + output[58] = step[23]; + output[59] = step[55]; + output[60] = step[15]; + output[61] = step[47]; + output[62] = step[31]; + output[63] = step[63]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs new file mode 100644 index 000000000..dee0fa89b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs @@ -0,0 +1,66 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the eight-point forward DCT stage network. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the eight-point forward discrete cosine transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static void Dct8( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + { + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + Av1ForwardTransformArithmetic.AddSubtract(input[0], input[7], out output[0], out output[7]); + Av1ForwardTransformArithmetic.AddSubtract(input[1], input[6], out output[1], out output[6]); + Av1ForwardTransformArithmetic.AddSubtract(input[2], input[5], out output[2], out output[5]); + Av1ForwardTransformArithmetic.AddSubtract(input[3], input[4], out output[3], out output[4]); + + // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + Av1ForwardTransformArithmetic.AddSubtract(output[0], output[3], out step[0], out step[3]); + Av1ForwardTransformArithmetic.AddSubtract(output[1], output[2], out step[1], out step[2]); + + step[4] = output[4]; + Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[5], output[6], out step[5], out step[6], cosBit, in rounding); + step[7] = output[7]; + + // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. + Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], step[0], step[1], out output[0], out output[1], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], step[3], step[2], out output[2], out output[3], cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(step[4], step[5], out output[4], out output[5]); + Av1ForwardTransformArithmetic.AddSubtract(step[7], step[6], out output[7], out output[6]); + + // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], output[7], output[4], out step[4], out step[7], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], output[6], output[5], out step[5], out step[6], cosBit, in rounding); + + // Stage 5 permutes the staged values into ascending AV1 coefficient order. + output[0] = step[0]; + output[1] = step[4]; + output[2] = step[2]; + output[3] = step[6]; + output[4] = step[1]; + output[5] = step[5]; + output[6] = step[3]; + output[7] = step[7]; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs new file mode 100644 index 000000000..73c4cea43 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs @@ -0,0 +1,111 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Implements the length-specific forward identity transform scaling. +/// +internal static partial class Av1ForwardTransformOperations +{ + /// + /// Applies the four-point forward identity transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The unused transform-stage buffer. + /// The unused fixed-point precision. + public static void Identity4( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Identity(ref input, ref output, ref step, cosBit, 4, 1, 0); + + /// + /// Applies the eight-point forward identity transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The unused transform-stage buffer. + /// The unused fixed-point precision. + public static void Identity8( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Identity(ref input, ref output, ref step, cosBit, 8, 0, 1); + + /// + /// Applies the sixteen-point forward identity transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The unused transform-stage buffer. + /// The unused fixed-point precision. + public static void Identity16( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Identity(ref input, ref output, ref step, cosBit, 16, 2, 0); + + /// + /// Applies the thirty-two-point forward identity transform to every independent lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The unused transform-stage buffer. + /// The unused fixed-point precision. + public static void Identity32( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Identity(ref input, ref output, ref step, cosBit, 32, 0, 2); + + /// + /// Applies the length-specific AV1 identity scaling to every transform value. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain values. + /// The frequency-domain values. + /// The unused transform-stage buffer. + /// The unused fixed-point precision. + /// The number of transform values. + /// The multiplier applied with the fixed-point square-root-of-two constant. + /// The direct left shift applied when square-root scaling is not required. + private static void Identity( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit, + int length, + int sqrt2Scale, + int leftShift) + where TValue : struct + { + _ = step; + _ = cosBit; + + // AV1 defines identity normalization by transform length: 4 and 16 use NewSqrt2 scaling, while 8 and 32 + // are exact powers of two. The same operation applies independently to each SIMD lane. + for (int i = 0; i < length; i++) + { + output[i] = sqrt2Scale != 0 + ? Av1ForwardTransformArithmetic.MultiplyRound( + input[i], + sqrt2Scale * Av1Transform1dMath.NewSqrt2, + Av1Transform1dMath.NewSqrt2Bits) + : Av1ForwardTransformArithmetic.ShiftLeft(input[i], leftShift); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs index 430baeb98..cd7dceca4 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs @@ -4,28 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 16-point AV1 forward identity transform operator. +/// Defines the sixteen-point AV1 forward identity transform operator. /// -internal readonly partial struct Av1Identity16Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Identity16Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 16-point AV1 forward identity transform. - /// - /// The sixteen spatial-domain residual values. - /// The sixteen scaled transform values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves sample order while applying a twice-square-root-of-two fixed-point scale for 2-D normalization. - for (int i = 0; i < 16; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); - } - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity16(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs index fc77d6c0b..0b96e7bff 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs @@ -4,28 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 32-point AV1 forward identity transform operator. +/// Defines the thirty-two-point AV1 forward identity transform operator. /// -internal readonly partial struct Av1Identity32Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Identity32Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 32-point AV1 forward identity transform. - /// - /// The 32 spatial-domain residual values. - /// The 32 scaled transform values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves sample order while applying an exact factor-of-four scale for 2-D normalization. - for (int i = 0; i < 32; i++) - { - output[i] = input[i] << 2; - } - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity32(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs index 45b6ccbe8..751a8218e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs @@ -4,28 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 4-point AV1 forward identity transform operator. +/// Defines the four-point AV1 forward identity transform operator. /// -internal readonly partial struct Av1Identity4Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Identity4Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 4-point AV1 forward identity transform. - /// - /// The four spatial-domain residual values. - /// The four scaled transform values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves sample order while applying a square-root-of-two fixed-point scale for 2-D normalization. - for (int i = 0; i < 4; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity4(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs index 697b6aaed..a330e6e8d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs @@ -4,28 +4,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// -/// Defines the 8-point AV1 forward identity transform operator. +/// Defines the eight-point AV1 forward identity transform operator. /// -internal readonly partial struct Av1Identity8Forward1dOperator : IAv1Transform1dOperator +internal readonly struct Av1Identity8Forward1dOperator : IAv1ForwardTransform1dOperator { - /// - /// Applies the normative 8-point AV1 forward identity transform. - /// - /// The eight spatial-domain residual values. - /// The eight scaled transform values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves sample order while applying an exact factor-of-two scale for 2-D normalization. - for (int i = 0; i < 8; i++) - { - output[i] = input[i] << 1; - } - } + /// + public static void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity8(ref input, ref output, ref step, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs deleted file mode 100644 index a8b362d03..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -/// -/// Provides the SIMD kernels for the four-point forward identity-transform operator. -/// -internal readonly partial struct Av1Identity4Forward1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } -} - -/// -/// Provides the SIMD kernels for the eight-point forward identity-transform operator. -/// -internal readonly partial struct Av1Identity8Forward1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} - -/// -/// Provides the SIMD kernels for the sixteen-point forward identity-transform operator. -/// -internal readonly partial struct Av1Identity16Forward1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } -} - -/// -/// Provides the SIMD kernels for the thirty-two-point forward identity-transform operator. -/// -internal readonly partial struct Av1Identity32Forward1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1TransformRounding.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1TransformRounding.cs new file mode 100644 index 000000000..7893511c8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1TransformRounding.cs @@ -0,0 +1,42 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Stores the fixed-point rounding value in the lane shape selected for one forward transform. +/// +/// +/// The fields overlap because a closed transform instantiation reads exactly one representation. This keeps the +/// rounding broadcast outside the butterfly sequence without increasing the caller-owned transform workspace. +/// +[StructLayout(LayoutKind.Explicit)] +internal struct Av1TransformRounding +{ + /// + /// The scalar rounding value. + /// + [FieldOffset(0)] + public int Scalar; + + /// + /// The four-lane rounding value used by 128-bit widened arithmetic. + /// + [FieldOffset(0)] + public Vector128 Vector128; + + /// + /// The eight-lane rounding value used by 256-bit widened arithmetic. + /// + [FieldOffset(0)] + public Vector256 Vector256; + + /// + /// The sixteen-lane rounding value used by 512-bit widened arithmetic. + /// + [FieldOffset(0)] + public Vector512 Vector512; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs new file mode 100644 index 000000000..8cf0e5f72 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs @@ -0,0 +1,30 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type. +/// +/// +/// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type +/// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch +/// inside the transform stages. +/// +internal interface IAv1ForwardTransform1dOperator +{ + /// + /// Transforms the independent axes stored in each value lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The spatial-domain transform values. + /// The frequency-domain transform values. + /// The fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static abstract void Transform( + ref Av1TransformVector input, + ref Av1TransformVector output, + ref Av1TransformVector step, + int cosBit) + where TValue : struct; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs index b88fff214..f9238be0c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs @@ -38,13 +38,4 @@ internal interface IAv1InverseTransformOutputOperator /// The eight inverse-transform residuals. /// The coded sample bit depth. public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); - - /// - /// Adds sixteen residuals to sixteen predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The sixteen inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector512 residual, int bitDepth); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs index e67a58d8b..81f39ccd0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs @@ -53,19 +53,4 @@ internal interface IAv1Transform1dOperator ref Av1TransformVector> step, int cosBit, Av1TransformStageRange stageRange); - - /// - /// Transforms sixteen independent axes in parallel. - /// - /// The source values for sixteen transform axes. - /// The destination values for sixteen transform axes. - /// The fixed stage storage for sixteen transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs deleted file mode 100644 index e472fdbde..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,569 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the sixteen-point inverse ADST operator. -/// -internal readonly partial struct Av1Adst16Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[15]; - output[1] = input[0]; - output[2] = input[13]; - output[3] = input[2]; - output[4] = input[11]; - output[5] = input[4]; - output[6] = input[9]; - output[7] = input[6]; - output[8] = input[7]; - output[9] = input[8]; - output[10] = input[5]; - output[11] = input[10]; - output[12] = input[3]; - output[13] = input[12]; - output[14] = input[1]; - output[15] = input[14]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[8]; - output[2] = step[12]; - output[3] = -step[4]; - output[4] = step[6]; - output[5] = -step[14]; - output[6] = step[10]; - output[7] = -step[2]; - output[8] = step[3]; - output[9] = -step[11]; - output[10] = step[15]; - output[11] = -step[7]; - output[12] = step[5]; - output[13] = -step[13]; - output[14] = step[9]; - output[15] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[15]; - output[1] = input[0]; - output[2] = input[13]; - output[3] = input[2]; - output[4] = input[11]; - output[5] = input[4]; - output[6] = input[9]; - output[7] = input[6]; - output[8] = input[7]; - output[9] = input[8]; - output[10] = input[5]; - output[11] = input[10]; - output[12] = input[3]; - output[13] = input[12]; - output[14] = input[1]; - output[15] = input[14]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[8]; - output[2] = step[12]; - output[3] = -step[4]; - output[4] = step[6]; - output[5] = -step[14]; - output[6] = step[10]; - output[7] = -step[2]; - output[8] = step[3]; - output[9] = -step[11]; - output[10] = step[15]; - output[11] = -step[7]; - output[12] = step[5]; - output[13] = -step[13]; - output[14] = step[9]; - output[15] = -step[1]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[15]; - output[1] = input[0]; - output[2] = input[13]; - output[3] = input[2]; - output[4] = input[11]; - output[5] = input[4]; - output[6] = input[9]; - output[7] = input[6]; - output[8] = input[7]; - output[9] = input[8]; - output[10] = input[5]; - output[11] = input[10]; - output[12] = input[3]; - output[13] = input[12]; - output[14] = input[1]; - output[15] = input[14]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[8]; - output[2] = step[12]; - output[3] = -step[4]; - output[4] = step[6]; - output[5] = -step[14]; - output[6] = step[10]; - output[7] = -step[2]; - output[8] = step[3]; - output[9] = -step[11]; - output[10] = step[15]; - output[11] = -step[7]; - output[12] = step[5]; - output[13] = -step[13]; - output[14] = step[9]; - output[15] = -step[1]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs index 3384fa035..c0d6b4345 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -190,4 +192,373 @@ internal readonly partial struct Av1Adst16Inverse1dOperator : IAv1Transform1dOpe output[14] = step[9]; output[15] = -step[1]; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs deleted file mode 100644 index 83a8a104f..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the four-point inverse ADST operator. -/// -internal readonly partial struct Av1Adst4Inverse1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit); - _ = step; - _ = stageRange; - } - - /// - /// Applies the inverse four-point matrix to four independent axes. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector128 x0 = input[0]; - Vector128 x1 = input[1]; - Vector128 x2 = input[2]; - Vector128 x3 = input[3]; - - // The products retain the sine-table scale across the complete matrix. The bounded transform inputs make - // the optimized kernels' wrapping 32-bit multiply/add sequence valid until the terminal rounding shift. - output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the inverse four-point matrix to eight independent axes. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector256 x0 = input[0]; - Vector256 x1 = input[1]; - Vector256 x2 = input[2]; - Vector256 x3 = input[3]; - - output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the inverse four-point matrix to sixteen independent axes. - /// - /// The source values for sixteen transform axes. - /// The destination values for sixteen transform axes. - /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector512 x0 = input[0]; - Vector512 x1 = input[1]; - Vector512 x2 = input[2]; - Vector512 x3 = input[3]; - - output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs index 7e1c9789b..86b0cbdd7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -64,4 +66,72 @@ internal readonly partial struct Av1Adst4Inverse1dOperator : IAv1Transform1dOper output[2] = Av1Math.RoundShift(x2, cosBit); output[3] = Av1Math.RoundShift(x3, cosBit); } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit); + _ = step; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit); + _ = step; + _ = stageRange; + } + + /// + /// Applies the inverse four-point matrix to four independent axes. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The fixed-point precision of the sine constants. + private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector128 x0 = input.V0; + Vector128 x1 = input.V1; + Vector128 x2 = input.V2; + Vector128 x3 = input.V3; + + // The products retain the sine-table scale across the complete matrix. The bounded transform inputs make + // the optimized kernels' wrapping 32-bit multiply/add sequence valid until the terminal rounding shift. + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } + + /// + /// Applies the inverse four-point matrix to eight independent axes. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The fixed-point precision of the sine constants. + private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector256 x0 = input.V0; + Vector256 x1 = input.V1; + Vector256 x2 = input.V2; + Vector256 x3 = input.V3; + + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs deleted file mode 100644 index a9b6df2fa..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the eight-point inverse ADST operator. -/// -internal readonly partial struct Av1Adst8Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[7]; - output[1] = input[0]; - output[2] = input[5]; - output[3] = input[2]; - output[4] = input[3]; - output[5] = input[4]; - output[6] = input[1]; - output[7] = input[6]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[4]; - output[2] = step[6]; - output[3] = -step[2]; - output[4] = step[3]; - output[5] = -step[7]; - output[6] = step[5]; - output[7] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[7]; - output[1] = input[0]; - output[2] = input[5]; - output[3] = input[2]; - output[4] = input[3]; - output[5] = input[4]; - output[6] = input[1]; - output[7] = input[6]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[4]; - output[2] = step[6]; - output[3] = -step[2]; - output[4] = step[3]; - output[5] = -step[7]; - output[6] = step[5]; - output[7] = -step[1]; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[7]; - output[1] = input[0]; - output[2] = input[5]; - output[3] = input[2]; - output[4] = input[3]; - output[5] = input[4]; - output[6] = input[1]; - output[7] = input[6]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[4]; - output[2] = step[6]; - output[3] = -step[2]; - output[4] = step[3]; - output[5] = -step[7]; - output[6] = step[5]; - output[7] = -step[1]; - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs index a4474bdf4..6a24b9841 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -97,4 +99,187 @@ internal readonly partial struct Av1Adst8Inverse1dOperator : IAv1Transform1dOper output[6] = step[5]; output[7] = -step[1]; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs deleted file mode 100644 index 1d6cdda62..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,476 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the sixteen-point inverse DCT operator. -/// -internal readonly partial struct Av1Dct16Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[8]; - output[2] = input[4]; - output[3] = input[12]; - output[4] = input[2]; - output[5] = input[10]; - output[6] = input[6]; - output[7] = input[14]; - output[8] = input[1]; - output[9] = input[9]; - output[10] = input[5]; - output[11] = input[13]; - output[12] = input[3]; - output[13] = input[11]; - output[14] = input[7]; - output[15] = input[15]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[8]; - output[2] = input[4]; - output[3] = input[12]; - output[4] = input[2]; - output[5] = input[10]; - output[6] = input[6]; - output[7] = input[14]; - output[8] = input[1]; - output[9] = input[9]; - output[10] = input[5]; - output[11] = input[13]; - output[12] = input[3]; - output[13] = input[11]; - output[14] = input[7]; - output[15] = input[15]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[8]; - output[2] = input[4]; - output[3] = input[12]; - output[4] = input[2]; - output[5] = input[10]; - output[6] = input[6]; - output[7] = input[14]; - output[8] = input[1]; - output[9] = input[9]; - output[10] = input[5]; - output[11] = input[13]; - output[12] = input[3]; - output[13] = input[11]; - output[14] = input[7]; - output[15] = input[15]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs index a96e7f3e6..e6f934498 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -159,4 +161,311 @@ internal readonly partial struct Av1Dct16Inverse1dOperator : IAv1Transform1dOper output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs deleted file mode 100644 index 3dc27a6ed..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,1028 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the thirty-two-point inverse DCT operator. -/// -internal readonly partial struct Av1Dct32Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[16]; - output[2] = input[8]; - output[3] = input[24]; - output[4] = input[4]; - output[5] = input[20]; - output[6] = input[12]; - output[7] = input[28]; - output[8] = input[2]; - output[9] = input[18]; - output[10] = input[10]; - output[11] = input[26]; - output[12] = input[6]; - output[13] = input[22]; - output[14] = input[14]; - output[15] = input[30]; - output[16] = input[1]; - output[17] = input[17]; - output[18] = input[9]; - output[19] = input[25]; - output[20] = input[5]; - output[21] = input[21]; - output[22] = input[13]; - output[23] = input[29]; - output[24] = input[3]; - output[25] = input[19]; - output[26] = input[11]; - output[27] = input[27]; - output[28] = input[7]; - output[29] = input[23]; - output[30] = input[15]; - output[31] = input[31]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); - output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); - output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); - output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); - output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); - output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); - output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); - output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); - output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); - step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); - step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); - step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); - step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); - step[31] = output[31]; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); - output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); - output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); - output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); - output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); - output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); - output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); - step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); - step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); - step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); - output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); - output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); - output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); - output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); - step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); - step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); - step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); - output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); - output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); - output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[16]; - output[2] = input[8]; - output[3] = input[24]; - output[4] = input[4]; - output[5] = input[20]; - output[6] = input[12]; - output[7] = input[28]; - output[8] = input[2]; - output[9] = input[18]; - output[10] = input[10]; - output[11] = input[26]; - output[12] = input[6]; - output[13] = input[22]; - output[14] = input[14]; - output[15] = input[30]; - output[16] = input[1]; - output[17] = input[17]; - output[18] = input[9]; - output[19] = input[25]; - output[20] = input[5]; - output[21] = input[21]; - output[22] = input[13]; - output[23] = input[29]; - output[24] = input[3]; - output[25] = input[19]; - output[26] = input[11]; - output[27] = input[27]; - output[28] = input[7]; - output[29] = input[23]; - output[30] = input[15]; - output[31] = input[31]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); - output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); - output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); - output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); - output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); - output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); - output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); - output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); - output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); - step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); - step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); - step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); - step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); - step[31] = output[31]; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); - output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); - output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); - output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); - output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); - output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); - output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); - step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); - step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); - step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); - output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); - output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); - output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); - output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); - step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); - step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); - step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); - output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); - output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); - output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[16]; - output[2] = input[8]; - output[3] = input[24]; - output[4] = input[4]; - output[5] = input[20]; - output[6] = input[12]; - output[7] = input[28]; - output[8] = input[2]; - output[9] = input[18]; - output[10] = input[10]; - output[11] = input[26]; - output[12] = input[6]; - output[13] = input[22]; - output[14] = input[14]; - output[15] = input[30]; - output[16] = input[1]; - output[17] = input[17]; - output[18] = input[9]; - output[19] = input[25]; - output[20] = input[5]; - output[21] = input[21]; - output[22] = input[13]; - output[23] = input[29]; - output[24] = input[3]; - output[25] = input[19]; - output[26] = input[11]; - output[27] = input[27]; - output[28] = input[7]; - output[29] = input[23]; - output[30] = input[15]; - output[31] = input[31]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); - output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); - output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); - output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); - output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); - output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); - output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); - output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); - output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); - step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); - step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); - step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); - step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); - step[31] = output[31]; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); - output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); - output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); - output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); - output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); - output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); - output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); - step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); - step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); - step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); - output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); - output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); - output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); - output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); - step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); - step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); - step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); - output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); - output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); - output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs index 0f44597ea..865070aad 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -343,4 +345,679 @@ internal readonly partial struct Av1Dct32Inverse1dOperator : IAv1Transform1dOper output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs deleted file mode 100644 index b56402464..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the four-point inverse DCT operator. -/// -internal readonly partial struct Av1Dct4Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output[0] = input[0]; - output[1] = input[2]; - output[2] = input[1]; - output[3] = input[3]; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output[0] = input[0]; - output[1] = input[2]; - output[2] = input[1]; - output[3] = input[3]; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output[0] = input[0]; - output[1] = input[2]; - output[2] = input[1]; - output[3] = input[3]; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs index ce04386bc..469699137 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -38,4 +40,69 @@ internal readonly partial struct Av1Dct4Inverse1dOperator : IAv1Transform1dOpera output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs deleted file mode 100644 index f83a6c811..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,2273 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the sixty-four-point inverse DCT operator. -/// -internal readonly partial struct Av1Dct64Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[32]; - output[2] = input[16]; - output[3] = input[48]; - output[4] = input[8]; - output[5] = input[40]; - output[6] = input[24]; - output[7] = input[56]; - output[8] = input[4]; - output[9] = input[36]; - output[10] = input[20]; - output[11] = input[52]; - output[12] = input[12]; - output[13] = input[44]; - output[14] = input[28]; - output[15] = input[60]; - output[16] = input[2]; - output[17] = input[34]; - output[18] = input[18]; - output[19] = input[50]; - output[20] = input[10]; - output[21] = input[42]; - output[22] = input[26]; - output[23] = input[58]; - output[24] = input[6]; - output[25] = input[38]; - output[26] = input[22]; - output[27] = input[54]; - output[28] = input[14]; - output[29] = input[46]; - output[30] = input[30]; - output[31] = input[62]; - output[32] = input[1]; - output[33] = input[33]; - output[34] = input[17]; - output[35] = input[49]; - output[36] = input[9]; - output[37] = input[41]; - output[38] = input[25]; - output[39] = input[57]; - output[40] = input[5]; - output[41] = input[37]; - output[42] = input[21]; - output[43] = input[53]; - output[44] = input[13]; - output[45] = input[45]; - output[46] = input[29]; - output[47] = input[61]; - output[48] = input[3]; - output[49] = input[35]; - output[50] = input[19]; - output[51] = input[51]; - output[52] = input[11]; - output[53] = input[43]; - output[54] = input[27]; - output[55] = input[59]; - output[56] = input[7]; - output[57] = input[39]; - output[58] = input[23]; - output[59] = input[55]; - output[60] = input[15]; - output[61] = input[47]; - output[62] = input[31]; - output[63] = input[63]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); - output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); - step[63] = output[63]; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); - step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); - step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); - step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); - step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); - step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); - step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); - output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); - output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); - output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); - output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); - output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); - output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); - output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); - output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); - output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[32]; - output[2] = input[16]; - output[3] = input[48]; - output[4] = input[8]; - output[5] = input[40]; - output[6] = input[24]; - output[7] = input[56]; - output[8] = input[4]; - output[9] = input[36]; - output[10] = input[20]; - output[11] = input[52]; - output[12] = input[12]; - output[13] = input[44]; - output[14] = input[28]; - output[15] = input[60]; - output[16] = input[2]; - output[17] = input[34]; - output[18] = input[18]; - output[19] = input[50]; - output[20] = input[10]; - output[21] = input[42]; - output[22] = input[26]; - output[23] = input[58]; - output[24] = input[6]; - output[25] = input[38]; - output[26] = input[22]; - output[27] = input[54]; - output[28] = input[14]; - output[29] = input[46]; - output[30] = input[30]; - output[31] = input[62]; - output[32] = input[1]; - output[33] = input[33]; - output[34] = input[17]; - output[35] = input[49]; - output[36] = input[9]; - output[37] = input[41]; - output[38] = input[25]; - output[39] = input[57]; - output[40] = input[5]; - output[41] = input[37]; - output[42] = input[21]; - output[43] = input[53]; - output[44] = input[13]; - output[45] = input[45]; - output[46] = input[29]; - output[47] = input[61]; - output[48] = input[3]; - output[49] = input[35]; - output[50] = input[19]; - output[51] = input[51]; - output[52] = input[11]; - output[53] = input[43]; - output[54] = input[27]; - output[55] = input[59]; - output[56] = input[7]; - output[57] = input[39]; - output[58] = input[23]; - output[59] = input[55]; - output[60] = input[15]; - output[61] = input[47]; - output[62] = input[31]; - output[63] = input[63]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); - output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); - step[63] = output[63]; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); - step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); - step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); - step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); - step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); - step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); - step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); - output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); - output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); - output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); - output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); - output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); - output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); - output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); - output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); - output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[32]; - output[2] = input[16]; - output[3] = input[48]; - output[4] = input[8]; - output[5] = input[40]; - output[6] = input[24]; - output[7] = input[56]; - output[8] = input[4]; - output[9] = input[36]; - output[10] = input[20]; - output[11] = input[52]; - output[12] = input[12]; - output[13] = input[44]; - output[14] = input[28]; - output[15] = input[60]; - output[16] = input[2]; - output[17] = input[34]; - output[18] = input[18]; - output[19] = input[50]; - output[20] = input[10]; - output[21] = input[42]; - output[22] = input[26]; - output[23] = input[58]; - output[24] = input[6]; - output[25] = input[38]; - output[26] = input[22]; - output[27] = input[54]; - output[28] = input[14]; - output[29] = input[46]; - output[30] = input[30]; - output[31] = input[62]; - output[32] = input[1]; - output[33] = input[33]; - output[34] = input[17]; - output[35] = input[49]; - output[36] = input[9]; - output[37] = input[41]; - output[38] = input[25]; - output[39] = input[57]; - output[40] = input[5]; - output[41] = input[37]; - output[42] = input[21]; - output[43] = input[53]; - output[44] = input[13]; - output[45] = input[45]; - output[46] = input[29]; - output[47] = input[61]; - output[48] = input[3]; - output[49] = input[35]; - output[50] = input[19]; - output[51] = input[51]; - output[52] = input[11]; - output[53] = input[43]; - output[54] = input[27]; - output[55] = input[59]; - output[56] = input[7]; - output[57] = input[39]; - output[58] = input[23]; - output[59] = input[55]; - output[60] = input[15]; - output[61] = input[47]; - output[62] = input[31]; - output[63] = input[63]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); - output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); - step[63] = output[63]; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); - step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); - step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); - step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); - step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); - step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); - step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); - output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); - output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); - output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); - output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); - output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); - output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); - output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); - output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); - output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs index 66840e0f6..34d274e04 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -758,4 +760,1509 @@ internal readonly partial struct Av1Dct64Inverse1dOperator : IAv1Transform1dOper output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs deleted file mode 100644 index 2531b3069..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the eight-point inverse DCT operator. -/// -internal readonly partial struct Av1Dct8Inverse1dOperator -{ - /// - /// Applies the transform to sixteen independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[4]; - output[2] = input[2]; - output[3] = input[6]; - output[4] = input[1]; - output[5] = input[5]; - output[6] = input[3]; - output[7] = input[7]; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[4]; - output[2] = input[2]; - output[3] = input[6]; - output[4] = input[1]; - output[5] = input[5]; - output[6] = input[3]; - output[7] = input[7]; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[4]; - output[2] = input[2]; - output[3] = input[6]; - output[4] = input[1]; - output[5] = input[5]; - output[6] = input[3]; - output[7] = input[7]; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs index 940344c3d..a79fda026 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -78,4 +80,149 @@ internal readonly partial struct Av1Dct8Inverse1dOperator : IAv1Transform1dOpera output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs index 53f535575..ec4f42d30 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -28,4 +30,32 @@ internal readonly partial struct Av1Identity16Inverse1dOperator : IAv1Transform1 output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); } } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs index c0fb5dcc9..0d0baf7a8 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -28,4 +30,32 @@ internal readonly partial struct Av1Identity32Inverse1dOperator : IAv1Transform1 output[i] = input[i] * 4; } } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs index e52279a77..a97b34086 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -28,4 +30,32 @@ internal readonly partial struct Av1Identity4Inverse1dOperator : IAv1Transform1d output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); } } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs index c75dd9857..c6029f1da 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; /// @@ -28,4 +30,32 @@ internal readonly partial struct Av1Identity8Inverse1dOperator : IAv1Transform1d output[i] = input[i] * 2; } } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs deleted file mode 100644 index ee3cea37e..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; - -/// -/// Provides the SIMD kernels for the four-point inverse identity-transform operator. -/// -internal readonly partial struct Av1Identity4Inverse1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } -} - -/// -/// Provides the SIMD kernels for the eight-point inverse identity-transform operator. -/// -internal readonly partial struct Av1Identity8Inverse1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} - -/// -/// Provides the SIMD kernels for the sixteen-point inverse identity-transform operator. -/// -internal readonly partial struct Av1Identity16Inverse1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - _ = step; - _ = cosBit; - _ = stageRange; - } -} - -/// -/// Provides the SIMD kernels for the thirty-two-point inverse identity-transform operator. -/// -internal readonly partial struct Av1Identity32Inverse1dOperator -{ - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } -} diff --git a/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs b/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs index 32949c667..9649282c1 100644 --- a/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs +++ b/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs @@ -4,341 +4,144 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; namespace SixLabors.ImageSharp.Benchmarks.Codecs.Heif; /// -/// Compares scalar, SIMD, and runtime-dispatched AV1 forward and inverse transform blocks. +/// Measures the production AV1 forward-transform pipeline at the runtime's selected vector width. /// +[Config(typeof(Configuration))] [MemoryDiagnoser(displayGenColumns: false)] [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] [CategoriesColumn] public class Av1TransformBenchmarks { - private readonly short[] spatial = new short[32 * 32]; + private readonly short[] spatial = new short[64 * 64]; private readonly int[] coefficients = new int[32 * 32]; - private readonly byte[] prediction = new byte[32 * 32]; - private readonly byte[] reconstruction = new byte[32 * 32]; private readonly int[] workspace = new int[Av1TransformWorkspace.MaximumLength]; /// - /// Initializes deterministic residual, coefficient, and prediction buffers outside the measured operations. + /// Gets or sets the coded sample bit depth used by the transform. + /// + [Params(8, 12)] + public int BitDepth { get; set; } + + /// + /// Initializes deterministic residual data outside the measured operations. /// [GlobalSetup] public void Setup() { for (int index = 0; index < this.spatial.Length; index++) { - this.spatial[index] = (short)(((index * 73) % 1023) - 511); - this.coefficients[index] = ((index * 37) % 129) - 64; - this.prediction[index] = (byte)(64 + ((index * 29) % 128)); + this.spatial[index] = (short)(((index * 73) % 511) - 255); } } /// - /// Measures the scalar eight-by-eight forward DCT traversal. - /// - /// The last coefficient written by the transform. - [Benchmark(Baseline = true)] - [BenchmarkCategory("Forward8x8")] - public int Forward8x8Scalar() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size8x8, 8); - Av1ForwardTransformer.Transform2dScalar( - this.spatial, this.coefficients, 8, ref config, this.workspace); - - return this.coefficients[63]; - } - - /// - /// Measures the Vector128 eight-by-eight forward DCT traversal. + /// Measures an eight-by-eight forward DCT block. /// /// The last coefficient written by the transform. [Benchmark] [BenchmarkCategory("Forward8x8")] - public int Forward8x8Vector128() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size8x8, 8); - Av1ForwardTransformer.Transform2dVector128( - this.spatial, this.coefficients, 8, ref config, this.workspace); - - return this.coefficients[63]; - } - - /// - /// Measures the Vector256 eight-by-eight forward DCT traversal. - /// - /// The last coefficient written by the transform. - [Benchmark] - [BenchmarkCategory("Forward8x8")] - public int Forward8x8Vector256() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size8x8, 8); - Av1ForwardTransformer.Transform2dVector256( - this.spatial, this.coefficients, 8, ref config, this.workspace); - - return this.coefficients[63]; - } - - /// - /// Measures runtime dispatch of an eight-by-eight forward DCT block. - /// - /// The last coefficient written by the transform. - [Benchmark] - [BenchmarkCategory("Forward8x8")] - public int Forward8x8Dispatch() + public int Forward8x8() { Av1ForwardTransformer.Transform2d( - this.spatial, this.coefficients, 8, Av1TransformType.DctDct, Av1TransformSize.Size8x8, 8, this.workspace); + this.spatial, + this.coefficients, + 8, + Av1TransformType.DctDct, + Av1TransformSize.Size8x8, + this.BitDepth, + this.workspace); return this.coefficients[63]; } /// - /// Measures the scalar thirty-two-by-thirty-two forward DCT traversal. - /// - /// The last coefficient written by the transform. - [Benchmark(Baseline = true)] - [BenchmarkCategory("Forward32x32")] - public int Forward32x32Scalar() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size32x32, 10); - Av1ForwardTransformer.Transform2dScalar( - this.spatial, this.coefficients, 32, ref config, this.workspace); - - return this.coefficients[^1]; - } - - /// - /// Measures the Vector128 thirty-two-by-thirty-two forward DCT traversal. - /// - /// The last coefficient written by the transform. - [Benchmark] - [BenchmarkCategory("Forward32x32")] - public int Forward32x32Vector128() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size32x32, 10); - Av1ForwardTransformer.Transform2dVector128( - this.spatial, this.coefficients, 32, ref config, this.workspace); - - return this.coefficients[^1]; - } - - /// - /// Measures the Vector256 thirty-two-by-thirty-two forward DCT traversal. - /// - /// The last coefficient written by the transform. - [Benchmark] - [BenchmarkCategory("Forward32x32")] - public int Forward32x32Vector256() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size32x32, 10); - Av1ForwardTransformer.Transform2dVector256( - this.spatial, this.coefficients, 32, ref config, this.workspace); - - return this.coefficients[^1]; - } - - /// - /// Measures the Vector512 thirty-two-by-thirty-two forward DCT traversal. - /// - /// The last coefficient written by the transform. - [Benchmark] - [BenchmarkCategory("Forward32x32")] - public int Forward32x32Vector512() - { - Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size32x32, 10); - Av1ForwardTransformer.Transform2dVector512( - this.spatial, this.coefficients, 32, ref config, this.workspace); - - return this.coefficients[^1]; - } - - /// - /// Measures runtime dispatch of a thirty-two-by-thirty-two forward DCT block. + /// Measures a thirty-two-by-thirty-two forward DCT block. /// /// The last coefficient written by the transform. [Benchmark] [BenchmarkCategory("Forward32x32")] - public int Forward32x32Dispatch() + public int Forward32x32() { Av1ForwardTransformer.Transform2d( - this.spatial, this.coefficients, 32, Av1TransformType.DctDct, Av1TransformSize.Size32x32, 10, this.workspace); + this.spatial, + this.coefficients, + 32, + Av1TransformType.DctDct, + Av1TransformSize.Size32x32, + this.BitDepth, + this.workspace); return this.coefficients[^1]; } /// - /// Measures the scalar eight-by-eight inverse DCT and byte reconstruction traversal. + /// Measures the packed-to-expanded boundary of a thirty-two-by-sixty-four forward DCT block. /// - /// The last reconstructed sample. - [Benchmark(Baseline = true)] - [BenchmarkCategory("Inverse8x8")] - public byte Inverse8x8Scalar() - { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size8x8, 8); - Av1Inverse2dTransformer.Transform2dScalar( - this.coefficients, this.prediction, 8, this.reconstruction, 8, ref config, this.workspace, 8); - - return this.reconstruction[63]; - } - - /// - /// Measures the Vector128 eight-by-eight inverse DCT and byte reconstruction traversal. - /// - /// The last reconstructed sample. + /// The last coded coefficient written by the transform. [Benchmark] - [BenchmarkCategory("Inverse8x8")] - public byte Inverse8x8Vector128() + [BenchmarkCategory("Forward32x64")] + public int Forward32x64() { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size8x8, 8); - Av1Inverse2dTransformer.Transform2dVector128( - this.coefficients, this.prediction, 8, this.reconstruction, 8, ref config, this.workspace, 8); - - return this.reconstruction[63]; - } - - /// - /// Measures the Vector256 eight-by-eight inverse DCT and byte reconstruction traversal. - /// - /// The last reconstructed sample. - [Benchmark] - [BenchmarkCategory("Inverse8x8")] - public byte Inverse8x8Vector256() - { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size8x8, 8); - Av1Inverse2dTransformer.Transform2dVector256( - this.coefficients, this.prediction, 8, this.reconstruction, 8, ref config, this.workspace, 8); - - return this.reconstruction[63]; - } - - /// - /// Measures runtime dispatch of an eight-by-eight inverse DCT and byte reconstruction block. - /// - /// The last reconstructed sample. - [Benchmark] - [BenchmarkCategory("Inverse8x8")] - public byte Inverse8x8Dispatch() - { - Av1InverseTransformer.Reconstruct8Bit( + Av1ForwardTransformer.Transform2d( + this.spatial, this.coefficients, - this.prediction, - 8, - this.reconstruction, - 8, - Av1TransformSize.Size8x8, + 32, Av1TransformType.DctDct, - 0, - 64, - false, + Av1TransformSize.Size32x64, + this.BitDepth, this.workspace); - return this.reconstruction[63]; - } - - /// - /// Measures the scalar thirty-two-by-thirty-two inverse DCT and byte reconstruction traversal. - /// - /// The last reconstructed sample. - [Benchmark(Baseline = true)] - [BenchmarkCategory("Inverse32x32")] - public byte Inverse32x32Scalar() - { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size32x32, 8); - Av1Inverse2dTransformer.Transform2dScalar( - this.coefficients, this.prediction, 32, this.reconstruction, 32, ref config, this.workspace, 8); - - return this.reconstruction[^1]; - } - - /// - /// Measures the Vector128 thirty-two-by-thirty-two inverse DCT and byte reconstruction traversal. - /// - /// The last reconstructed sample. - [Benchmark] - [BenchmarkCategory("Inverse32x32")] - public byte Inverse32x32Vector128() - { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size32x32, 8); - Av1Inverse2dTransformer.Transform2dVector128( - this.coefficients, this.prediction, 32, this.reconstruction, 32, ref config, this.workspace, 8); - - return this.reconstruction[^1]; - } - - /// - /// Measures the Vector256 thirty-two-by-thirty-two inverse DCT and byte reconstruction traversal. - /// - /// The last reconstructed sample. - [Benchmark] - [BenchmarkCategory("Inverse32x32")] - public byte Inverse32x32Vector256() - { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size32x32, 8); - Av1Inverse2dTransformer.Transform2dVector256( - this.coefficients, this.prediction, 32, this.reconstruction, 32, ref config, this.workspace, 8); - - return this.reconstruction[^1]; - } - - /// - /// Measures the Vector512 thirty-two-by-thirty-two inverse DCT and byte reconstruction traversal. - /// - /// The last reconstructed sample. - [Benchmark] - [BenchmarkCategory("Inverse32x32")] - public byte Inverse32x32Vector512() - { - Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size32x32, 8); - Av1Inverse2dTransformer.Transform2dVector512( - this.coefficients, this.prediction, 32, this.reconstruction, 32, ref config, this.workspace, 8); - - return this.reconstruction[^1]; + return this.coefficients[^1]; } /// - /// Measures runtime dispatch of a thirty-two-by-thirty-two inverse DCT and byte reconstruction block. + /// Measures a sixty-four-by-sixty-four forward DCT block with the normative coefficient truncation. /// - /// The last reconstructed sample. + /// The last coded coefficient written by the transform. [Benchmark] - [BenchmarkCategory("Inverse32x32")] - public byte Inverse32x32Dispatch() + [BenchmarkCategory("Forward64x64")] + public int Forward64x64() { - Av1InverseTransformer.Reconstruct8Bit( + Av1ForwardTransformer.Transform2d( + this.spatial, this.coefficients, - this.prediction, - 32, - this.reconstruction, - 32, - Av1TransformSize.Size32x32, + 64, Av1TransformType.DctDct, - 0, - 1024, - false, + Av1TransformSize.Size64x64, + this.BitDepth, this.workspace); - return this.reconstruction[^1]; + return this.coefficients[^1]; } /// - /// Creates a forward DCT configuration with the normative stage ranges for one coded bit depth. - /// - /// The dimensions of the transform block. - /// The coded sample bit depth. - /// The initialized transform configuration. - private static Av1Transform2dFlipConfiguration CreateForwardConfiguration(Av1TransformSize transformSize, int bitDepth) - => Av1Transform2dFlipConfiguration.CreateForward(Av1TransformType.DctDct, transformSize, bitDepth); - - /// - /// Creates an inverse DCT configuration with the normative stage ranges for one coded bit depth. + /// Configures separate production-process measurements for preferred 256-bit and 512-bit vectors. /// - /// The dimensions of the transform block. - /// The coded sample bit depth. - /// The initialized transform configuration. - private static Av1Transform2dFlipConfiguration CreateInverseConfiguration(Av1TransformSize transformSize, int bitDepth) - => Av1Transform2dFlipConfiguration.CreateInverse(Av1TransformType.DctDct, transformSize, bitDepth); + public sealed class Configuration : ManualConfig + { + /// + /// Initializes a new instance of the class. + /// + public Configuration() + { + this.AddJob( + Job.ShortRun + .WithId("Vector256") + .WithEnvironmentVariable("DOTNET_PreferredVectorBitWidth", "256") + .WithEnvironmentVariable("COMPlus_PreferredVectorBitWidth", "256") + .AsBaseline()); + + this.AddJob( + Job.ShortRun + .WithId("Vector512") + .WithEnvironmentVariable("DOTNET_PreferredVectorBitWidth", "512") + .WithEnvironmentVariable("COMPlus_PreferredVectorBitWidth", "512")); + } + } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index ae660a562..b29624536 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -2,6 +2,8 @@ // Licensed under the Six Labors Split License. using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; using SixLabors.ImageSharp.Tests.TestUtilities; @@ -12,96 +14,33 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; public class Av1ForwardTransformTests { /// - /// The hardware configurations covering every transform SIMD tier and the scalar fallback. + /// The hardware configurations covering every transform vector tier and the scalar fallback. /// private const HwIntrinsics TransformConfigurations = HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; /// - /// Gets every normative transform size, type, and bit-depth combination exercised by the forward and inverse suites. + /// Gets every normative transform size, type, and bit-depth combination shared with the inverse suite. /// public static TheoryData ValidTransformCases { get; } = CreateValidTransformCases(); /// - /// Verifies DCT operator parity across the supported hardware feature levels. + /// Verifies every one-dimensional stage network across its scalar and available vector representations. /// [Fact] - public void DctOperatorsProduceIdenticalScalarAndSimdResults() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertDctOperatorParity, TransformConfigurations); + public void OneDimensionalOperatorsMatchAcrossHardwareWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertOneDimensionalOperators, TransformConfigurations); /// - /// Verifies ADST operator parity across the supported hardware feature levels. + /// Verifies every permitted size, type, and bit-depth combination against the direct scalar two-axis definition. /// [Fact] - public void AdstOperatorsProduceIdenticalScalarAndSimdResults() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertAdstOperatorParity, TransformConfigurations); + public void TwoDimensionalPipelineMatchesScalarReferenceAcrossHardwareConfigurations() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertTwoDimensionalPipeline, TransformConfigurations); /// - /// Verifies identity operator parity across the supported hardware feature levels. + /// Verifies that the complete transform dispatcher reuses caller-owned workspace. /// - [Fact] - public void IdentityOperatorsProduceIdenticalScalarAndSimdResults() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertIdentityOperatorParity, TransformConfigurations); - - /// - /// Verifies the complete sixteen-lane two-dimensional traversal matrix across hardware feature levels. - /// - [Fact] - public void Vector512KernelsMatchScalarForEveryApplicableConfiguration() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertVector512TransformParity, TransformConfigurations); - - /// - /// Verifies the forward DCT operators against their scalar implementations. - /// - private static void AssertDctOperatorParity() - { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); - AssertOperatorParity(64); - } - - /// - /// Verifies the forward ADST operators against their scalar implementations. - /// - private static void AssertAdstOperatorParity() - { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - } - - /// - /// Verifies the forward identity operators against their scalar implementations. - /// - private static void AssertIdentityOperatorParity() - { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); - } - - /// - /// Verifies that every applicable SIMD traversal produces the same coefficients as the scalar traversal. - /// - /// The integral value. - /// The integral value. - /// The coded sample bit depth. - [Theory] - [MemberData(nameof(ValidTransformCases))] - public void TwoDimensionalSimdKernelsMatchScalarForEveryValidConfiguration( - int transformTypeValue, - int transformSizeValue, - int bitDepth) - { - Av1TransformType transformType = (Av1TransformType)transformTypeValue; - Av1TransformSize transformSize = (Av1TransformSize)transformSizeValue; - Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); - DispatchColumn(transformType, transformSize, bitDepth, ref config); - } - [Fact] public void TransformDispatchDoesNotAllocatePerBlock() { @@ -118,145 +57,158 @@ public class Av1ForwardTransformTests Av1ForwardTransformer.Transform2d(input, output, width, Av1TransformType.DctDct, Av1TransformSize.Size8x8, 8, workspace); } - long allocated = GC.GetAllocatedBytesForCurrentThread() - before; - Assert.Equal(0, allocated); + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + } + + /// + /// Exercises every DCT, ADST, and identity stage network using both Int16 and Int32 lane arithmetic. + /// + private static void AssertOneDimensionalOperators() + { + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); + AssertOperator(64); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); } /// - /// Compares one forward transform operator across scalar and all SIMD lane widths. + /// Compares one stage network across all available scalar and vector representations. /// - /// The forward transform operator. + /// The transform operator. /// The transform length. - private static void AssertOperatorParity(int length) - where TOperator : struct, IAv1Transform1dOperator + private static void AssertOperator(int length) + where TOperator : struct, IAv1ForwardTransform1dOperator { const int cosBit = 12; - Av1TransformStageRange stageRange = default; - for (int index = 0; index < Av1Transform2dFlipConfiguration.MaxStageNumber; index++) + AssertInt32Operator>(length, cosBit); + + if (Vector256.IsHardwareAccelerated) { - stageRange[index] = 24; + AssertInt32Operator>(length, cosBit); } - Av1TransformVector> input128 = default; - Av1TransformVector> output128 = default; - Av1TransformVector> step128 = default; - Av1TransformVector> input256 = default; - Av1TransformVector> output256 = default; - Av1TransformVector> step256 = default; - Av1TransformVector> input512 = default; - Av1TransformVector> output512 = default; - Av1TransformVector> step512 = default; - - for (int index = 0; index < length; index++) + if (Vector512.IsHardwareAccelerated) { - input128[index] = Vector128.Create( - GetInputValue(index, 0), - GetInputValue(index, 1), - GetInputValue(index, 2), - GetInputValue(index, 3)); - - input256[index] = Vector256.Create( - GetInputValue(index, 0), - GetInputValue(index, 1), - GetInputValue(index, 2), - GetInputValue(index, 3), - GetInputValue(index, 4), - GetInputValue(index, 5), - GetInputValue(index, 6), - GetInputValue(index, 7)); - - input512[index] = Vector512.Create( - GetInputValue(index, 0), - GetInputValue(index, 1), - GetInputValue(index, 2), - GetInputValue(index, 3), - GetInputValue(index, 4), - GetInputValue(index, 5), - GetInputValue(index, 6), - GetInputValue(index, 7), - GetInputValue(index, 8), - GetInputValue(index, 9), - GetInputValue(index, 10), - GetInputValue(index, 11), - GetInputValue(index, 12), - GetInputValue(index, 13), - GetInputValue(index, 14), - GetInputValue(index, 15)); + AssertInt32Operator>(length, cosBit); + } + AssertInt16Operator>(length, cosBit); + + if (Avx2.IsSupported) + { + AssertInt16Operator>(length, cosBit); } - TOperator.Transform(ref input128, ref output128, ref step128, cosBit, stageRange); - TOperator.Transform(ref input256, ref output256, ref step256, cosBit, stageRange); - TOperator.Transform(ref input512, ref output512, ref step512, cosBit, stageRange); + if (Avx512BW.IsSupported) + { + AssertInt16Operator>(length, cosBit); + } + } - int[] scalarInput = new int[length]; - int[] scalarOutput = new int[length]; - int[] scalarStep = new int[length]; + /// + /// Compares one Int32 vector representation with the scalar Int32 stage network lane by lane. + /// + private static void AssertInt32Operator(int length, int cosBit) + where TOperator : struct, IAv1ForwardTransform1dOperator + where TVector : struct + { + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(int); + Av1TransformVector vectorValues = default; + Av1TransformVector vectorBuffer0 = default; + Av1TransformVector vectorBuffer1 = default; - for (int lane = 0; lane < Vector512.Count; lane++) + for (int index = 0; index < length; index++) { - for (int index = 0; index < length; index++) + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + + for (int lane = 0; lane < laneCount; lane++) { - scalarInput[index] = GetInputValue(index, lane); + System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetInputValue(index, lane); } + } - TOperator.Transform(scalarInput, scalarOutput, scalarStep, cosBit, stageRange); + TOperator.Transform(ref vectorValues, ref vectorBuffer0, ref vectorBuffer1, cosBit); + + for (int lane = 0; lane < laneCount; lane++) + { + Av1TransformVector scalarValues = default; + Av1TransformVector scalarBuffer0 = default; + Av1TransformVector scalarBuffer1 = default; for (int index = 0; index < length; index++) { - Assert.Equal(scalarOutput[index], output512[index].GetElement(lane)); + scalarValues[index] = GetInputValue(index, lane); + } - if (lane < Vector256.Count) - { - Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); - } + TOperator.Transform(ref scalarValues, ref scalarBuffer0, ref scalarBuffer1, cosBit); - if (lane < Vector128.Count) - { - Assert.Equal(scalarOutput[index], output128[index].GetElement(lane)); - } + for (int index = 0; index < length; index++) + { + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorBuffer0[index]); + Assert.Equal(scalarBuffer0[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } /// - /// Runs every valid forward transform configuration capable of filling a sixteen-lane tile. + /// Compares one Int16 vector representation with the scalar Int16 stage network lane by lane. /// - private static void AssertVector512TransformParity() + private static void AssertInt16Operator(int length, int cosBit) + where TOperator : struct, IAv1ForwardTransform1dOperator + where TVector : struct { - for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) + int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(short); + Av1TransformVector vectorValues = default; + Av1TransformVector vectorBuffer0 = default; + Av1TransformVector vectorBuffer1 = default; + + for (int index = 0; index < length; index++) { - if (transformSize.GetWidth() < Vector512.Count || transformSize.GetHeight() < Vector512.Count) + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + + for (int lane = 0; lane < laneCount; lane++) { - continue; + System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane) = GetPackedInputValue(index, lane); } + } - for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) + TOperator.Transform(ref vectorValues, ref vectorBuffer0, ref vectorBuffer1, cosBit); + + for (int lane = 0; lane < laneCount; lane++) + { + Av1TransformVector scalarValues = default; + Av1TransformVector scalarBuffer0 = default; + Av1TransformVector scalarBuffer1 = default; + + for (int index = 0; index < length; index++) { - Av1Transform2dFlipConfiguration allowedConfig = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, 8); - if (!allowedConfig.IsAllowed()) - { - continue; - } + scalarValues[index] = GetPackedInputValue(index, lane); + } - for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) - { - Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); - DispatchColumn(transformType, transformSize, bitDepth, ref config); - } + TOperator.Transform(ref scalarValues, ref scalarBuffer0, ref scalarBuffer1, cosBit); + + for (int index = 0; index < length; index++) + { + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorBuffer0[index]); + Assert.Equal(scalarBuffer0[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } /// - /// Creates the complete normative transform matrix shared by the forward and inverse parity tests. + /// Exercises the complete normative transform matrix for the active hardware configuration. /// - /// The transform type, size, and bit-depth cases. - private static TheoryData CreateValidTransformCases() + private static void AssertTwoDimensionalPipeline() { - TheoryData cases = []; - for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) { for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) @@ -268,209 +220,247 @@ public class Av1ForwardTransformTests continue; } - // libaom verifies the low-bit-depth kernel separately from its 10- and 12-bit kernels. Keeping each - // depth as a distinct case makes any fixed-point range failure identify the exact configuration. for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) { - cases.Add((int)transformType, (int)transformSize, bitDepth); + AssertTwoDimensionalCase(transformType, transformSize, bitDepth); } } } + } - return cases; + /// + /// Compares one complete transform with the direct scalar two-axis definition. + /// + private static void AssertTwoDimensionalCase(Av1TransformType transformType, Av1TransformSize transformSize, int bitDepth) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + int inputStride = width + 3; + Av1TransformSize adjustedSize = transformSize.GetAdjusted(); + int coefficientCount = adjustedSize.GetWidth() * adjustedSize.GetHeight(); + short[] input = new short[inputStride * height]; + int sampleMaximum = (1 << bitDepth) - 1; + + for (int row = 0; row < height; row++) + { + for (int column = 0; column < width; column++) + { + int index = (row * width) + column; + input[(row * inputStride) + column] = (short)((index & 3) switch + { + 0 => sampleMaximum, + 1 => -sampleMaximum, + 2 => ((index * 73) % ((2 * sampleMaximum) + 1)) - sampleMaximum, + _ => 0, + }); + } + } + + int[] expected = new int[coefficientCount]; + int[] actual = new int[coefficientCount]; + int[] workspace = new int[Av1TransformWorkspace.MaximumLength]; + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); + + DispatchReferenceColumn(input, inputStride, expected, ref config); + Av1ForwardTransformer.Transform2d(input, actual, (uint)inputStride, transformType, transformSize, bitDepth, workspace); + Assert.Equal(expected, actual); } /// - /// Closes the static-generic column operator selected by a transform configuration. + /// Selects the scalar reference column operator. /// - /// The compound transform type. - /// The transform-block dimensions. - /// The coded sample bit depth. - /// The forward transform configuration. - private static void DispatchColumn( - Av1TransformType transformType, - Av1TransformSize transformSize, - int bitDepth, - ref Av1Transform2dFlipConfiguration config) + private static void DispatchReferenceColumn(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) { switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchRow(transformType, transformSize, bitDepth, ref config); - break; - default: - Assert.Fail($"Unexpected column function {config.TransformFunctionTypeColumn} for {transformType} {transformSize}."); + DispatchReferenceRow(input, stride, output, ref config); break; } } /// - /// Closes the static-generic row operator after the column operator has been selected. + /// Selects the scalar reference row operator. /// - /// The selected column operator. - /// The compound transform type. - /// The transform-block dimensions. - /// The coded sample bit depth. - /// The forward transform configuration. - private static void DispatchRow( - Av1TransformType transformType, - Av1TransformSize transformSize, - int bitDepth, - ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator + private static void DispatchReferenceRow(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) + where TColumnOperator : struct, IAv1ForwardTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - AssertTransform2dParity(transformSize, bitDepth, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - AssertTransform2dParity(transformSize, bitDepth, ref config); - break; - default: - Assert.Fail($"Unexpected row function {config.TransformFunctionTypeRow} for {transformType} {transformSize}."); + TransformReference(input, stride, output, ref config); break; } } /// - /// Compares scalar and SIMD forward traversals using padded rows and bounded extreme residuals. + /// Applies the direct scalar column and row transform definition used as the layout and dispatch oracle. /// - /// The selected column operator. - /// The selected row operator. - /// The transform-block dimensions. - /// The coded sample bit depth. - /// The forward transform configuration. - private static void AssertTransform2dParity( - Av1TransformSize transformSize, - int bitDepth, + private static void TransformReference( + Span input, + int stride, + Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, IAv1Transform1dOperator - where TRowOperator : struct, IAv1Transform1dOperator + where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TRowOperator : struct, IAv1ForwardTransform1dOperator { - int width = transformSize.GetWidth(); - int height = transformSize.GetHeight(); - int inputStride = width + 5; - int maximum = (1 << bitDepth) - 1; - short[] input = new short[inputStride * height]; + int width = config.TransformSize.GetWidth(); + int height = config.TransformSize.GetHeight(); + int outputWidth = Math.Min(width, 32); + int outputHeight = Math.Min(height, 32); + int[] intermediate = new int[width * height]; + Av1TransformVector values = default; + Av1TransformVector buffer0 = default; + Av1TransformVector buffer1 = default; + + for (int column = 0; column < width; column++) + { + for (int row = 0; row < height; row++) + { + int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; + values[row] = input[(sourceRow * stride) + column] << config.Shift0; + } - // Padded rows exercise the same edge-block layout used by the encoder. The alternating extrema are the - // bounded residual limits used by libaom's SIMD match tests and expose wrapping errors in fixed-point stages. - for (int row = 0; row < height; row++) + TColumnOperator.Transform(ref values, ref buffer0, ref buffer1, config.CosBitColumn); + int destinationColumn = config.FlipLeftToRight ? width - column - 1 : column; + + for (int row = 0; row < height; row++) + { + intermediate[(row * width) + destinationColumn] = Av1Math.RoundShift(buffer0[row], -config.Shift1); + } + } + + bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; + + for (int row = 0; row < outputHeight; row++) { for (int column = 0; column < width; column++) { - int index = (row * width) + column; - input[(row * inputStride) + column] = (short)((index & 3) switch - { - 0 => maximum, - 1 => -maximum, - 2 => ((index * 73) % ((maximum * 2) + 1)) - maximum, - _ => 0, - }); + values[column] = intermediate[(row * width) + column]; } - } - int coefficientCount = width * height; - int workspaceLength = Av1TransformWorkspace.GetRequiredLength(transformSize); - int[] scalar = new int[coefficientCount]; - int[] vector128 = new int[coefficientCount]; - int[] scalarWorkspace = new int[workspaceLength]; - int[] vector128Workspace = new int[workspaceLength]; + TRowOperator.Transform(ref values, ref buffer0, ref buffer1, config.CosBitRow); - Av1ForwardTransformer.Transform2dScalar(input, scalar, (uint)inputStride, ref config, scalarWorkspace); - Av1ForwardTransformer.Transform2dVector128(input, vector128, (uint)inputStride, ref config, vector128Workspace); + for (int column = 0; column < outputWidth; column++) + { + int value = Av1Math.RoundShift(buffer0[column], -config.Shift2); + output[(row * outputWidth) + column] = normalizeRectangle + ? Av1Transform1dMath.HalfButterfly(Av1Transform1dMath.NewSqrt2, value, 0, 0, Av1Transform1dMath.NewSqrt2Bits) + : value; + } + } + } - Assert.Equal(scalar, vector128); + /// + /// Gets a deterministic signed thirty-two-bit transform input. + /// + private static int GetInputValue(int index, int lane) + => (((index * 73) + (lane * 151)) % 8191) - 4095; - // The production dispatcher uses 256-bit lanes only when both axes contain a complete eight-lane tile. - if (width >= Vector256.Count && height >= Vector256.Count) + /// + /// Gets a deterministic signed sixteen-bit input including overflow-sensitive edge values. + /// + private static short GetPackedInputValue(int index, int lane) + => (short)((index + lane) % 5 switch { - int[] vector256 = new int[coefficientCount]; - int[] vector256Workspace = new int[workspaceLength]; - Av1ForwardTransformer.Transform2dVector256(input, vector256, (uint)inputStride, ref config, vector256Workspace); + 0 => short.MaxValue, + 1 => short.MinValue, + 2 => 255, + 3 => -255, + _ => ((index * 73) + (lane * 151)) % 511 - 255, + }); - Assert.Equal(scalar, vector256); - } + /// + /// Creates the complete normative transform matrix shared by the forward and inverse tests. + /// + private static TheoryData CreateValidTransformCases() + { + TheoryData cases = []; - if (width >= Vector512.Count && height >= Vector512.Count) + for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) { - int[] vector512 = new int[coefficientCount]; - int[] vector512Workspace = new int[workspaceLength]; - Av1ForwardTransformer.Transform2dVector512(input, vector512, (uint)inputStride, ref config, vector512Workspace); + for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) + { + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, 8); + + if (!config.IsAllowed()) + { + continue; + } - Assert.Equal(scalar, vector512); + for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) + { + cases.Add((int)transformType, (int)transformSize, bitDepth); + } + } } - } - /// - /// Produces deterministic bounded input for one transform position and SIMD lane. - /// - /// The position within the transform. - /// The SIMD lane index. - /// The input value. - private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 1023) - 511; + return cases; + } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 5674158cc..dc5883e2e 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -73,13 +73,6 @@ public class Av1InverseTransformTests AssertOperatorParity(32); } - /// - /// Verifies the complete sixteen-lane two-dimensional inverse traversal matrix across hardware feature levels. - /// - [Fact] - public void Vector512KernelsMatchScalarForEveryApplicableConfiguration() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertVector512TransformParity, TransformConfigurations); - [Theory] [InlineData((int)Av1TransformSize.Size4x4, 0, -4)] [InlineData((int)Av1TransformSize.Size8x8, -1, -4)] @@ -251,7 +244,7 @@ public class Av1InverseTransformTests } /// - /// Compares one inverse transform operator across scalar and all SIMD lane widths. + /// Compares one inverse transform operator across scalar and the supported SIMD lane widths. /// /// The inverse transform operator. /// The transform length. @@ -272,9 +265,6 @@ public class Av1InverseTransformTests Av1TransformVector> input256 = default; Av1TransformVector> output256 = default; Av1TransformVector> step256 = default; - Av1TransformVector> input512 = default; - Av1TransformVector> output512 = default; - Av1TransformVector> step512 = default; for (int index = 0; index < length; index++) { @@ -293,36 +283,16 @@ public class Av1InverseTransformTests GetInputValue(index, 5), GetInputValue(index, 6), GetInputValue(index, 7)); - - input512[index] = Vector512.Create( - GetInputValue(index, 0), - GetInputValue(index, 1), - GetInputValue(index, 2), - GetInputValue(index, 3), - GetInputValue(index, 4), - GetInputValue(index, 5), - GetInputValue(index, 6), - GetInputValue(index, 7), - GetInputValue(index, 8), - GetInputValue(index, 9), - GetInputValue(index, 10), - GetInputValue(index, 11), - GetInputValue(index, 12), - GetInputValue(index, 13), - GetInputValue(index, 14), - GetInputValue(index, 15)); - } TOperator.Transform(ref input128, ref output128, ref step128, cosBit, stageRange); TOperator.Transform(ref input256, ref output256, ref step256, cosBit, stageRange); - TOperator.Transform(ref input512, ref output512, ref step512, cosBit, stageRange); int[] scalarInput = new int[length]; int[] scalarOutput = new int[length]; int[] scalarStep = new int[length]; - for (int lane = 0; lane < Vector512.Count; lane++) + for (int lane = 0; lane < Vector256.Count; lane++) { for (int index = 0; index < length; index++) { @@ -333,12 +303,7 @@ public class Av1InverseTransformTests for (int index = 0; index < length; index++) { - Assert.Equal(scalarOutput[index], output512[index].GetElement(lane)); - - if (lane < Vector256.Count) - { - Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); - } + Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); if (lane < Vector128.Count) { @@ -348,35 +313,6 @@ public class Av1InverseTransformTests } } - /// - /// Runs every valid inverse transform configuration capable of filling a sixteen-lane tile. - /// - private static void AssertVector512TransformParity() - { - for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) - { - if (transformSize.GetWidth() < Vector512.Count || transformSize.GetHeight() < Vector512.Count) - { - continue; - } - - for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) - { - Av1Transform2dFlipConfiguration allowedConfig = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, 8); - if (!allowedConfig.IsAllowed()) - { - continue; - } - - for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) - { - Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, bitDepth); - DispatchColumn(transformType, transformSize, bitDepth, ref config); - } - } - } - } - /// /// Verifies that a matching one-dimensional forward and inverse operator pair reconstructs bounded input. /// @@ -387,7 +323,7 @@ public class Av1InverseTransformTests /// The power-of-two scale applied by the operator pair. /// The maximum permitted reconstruction error. private static void AssertRoundTrip(Av1TransformType transformType, Av1TransformSize transformSize, int scaleLog2, int allowedError) - where TForwardOperator : struct, IAv1Transform1dOperator + where TForwardOperator : struct, IAv1ForwardTransform1dOperator where TInverseOperator : struct, IAv1Transform1dOperator { const int bitDepth = 10; @@ -400,15 +336,25 @@ public class Av1InverseTransformTests int[] forward = new int[length]; int[] inverse = new int[length]; int[] step = new int[length]; + Av1TransformVector values = default; + Av1TransformVector buffer0 = default; + Av1TransformVector buffer1 = default; for (int block = 0; block < testBlockCount; block++) { for (int index = 0; index < length; index++) { input[index] = random.Next((1 << bitDepth) - 1); + values[index] = input[index]; + } + + TForwardOperator.Transform(ref values, ref buffer0, ref buffer1, forwardConfig.CosBitColumn); + + for (int index = 0; index < length; index++) + { + forward[index] = buffer0[index]; } - TForwardOperator.Transform(input, forward, step, forwardConfig.CosBitColumn, forwardConfig.StageRangeColumn); TInverseOperator.Transform(forward, inverse, step, inverseConfig.CosBitColumn, inverseConfig.StageRangeColumn); for (int index = 0; index < length; index++) @@ -645,18 +591,6 @@ public class Av1InverseTransformTests Assert.Equal(scalar, vector256); } - - if (width >= Vector512.Count && height >= Vector512.Count) - { - byte[] vector512 = new byte[writeStride * height]; - int[] vector512Workspace = new int[workspaceLength]; - Array.Fill(vector512, byte.MaxValue); - - Av1Inverse2dTransformer.Transform2dVector512( - coefficients, prediction, readStride, vector512, writeStride, ref config, vector512Workspace, bitDepth); - - Assert.Equal(scalar, vector512); - } } /// @@ -718,18 +652,6 @@ public class Av1InverseTransformTests Assert.Equal(scalar, vector256); } - - if (width >= Vector512.Count && height >= Vector512.Count) - { - short[] vector512 = new short[writeStride * height]; - int[] vector512Workspace = new int[workspaceLength]; - Array.Fill(vector512, short.MinValue); - - Av1Inverse2dTransformer.Transform2dVector512( - coefficients, prediction, readStride, vector512, writeStride, ref config, vector512Workspace, bitDepth); - - Assert.Equal(scalar, vector512); - } } ///