From 86d44f158af66e6d105a39f1fc2854c56270d8bb Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sun, 30 Aug 2026 00:42:17 +1000 Subject: [PATCH] Restore AV1 transform operator architecture --- HEIF_IMPLEMENTATION_PLAN.md | 2 +- .../Av1ForwardTransformer.Adst16Operator.cs | 29 - .../Av1ForwardTransformer.Adst4Operator.cs | 29 - .../Av1ForwardTransformer.Adst8Operator.cs | 29 - .../Av1ForwardTransformer.Dct16Operator.cs | 29 - .../Av1ForwardTransformer.Dct32Operator.cs | 29 - .../Av1ForwardTransformer.Dct4Operator.cs | 29 - .../Av1ForwardTransformer.Dct64Operator.cs | 29 - .../Av1ForwardTransformer.Dct8Operator.cs | 29 - ...v1ForwardTransformer.Identity16Operator.cs | 29 - ...v1ForwardTransformer.Identity32Operator.cs | 29 - ...Av1ForwardTransformer.Identity4Operator.cs | 29 - ...Av1ForwardTransformer.Identity8Operator.cs | 29 - ...r.Operator.cs => Av1ForwardTransformer.cs} | 80 +- ...Operator.cs => Av1Inverse2dTransformer.cs} | 171 +- .../Av1InverseTransformOutputOperator.cs | 94 + .../Av1InverseTransformer.Adst16Operator.cs | 574 ----- .../Av1InverseTransformer.Adst4Operator.cs | 174 -- .../Av1InverseTransformer.Adst8Operator.cs | 295 --- .../Av1InverseTransformer.Dct16Operator.cs | 481 ---- .../Av1InverseTransformer.Dct32Operator.cs | 1033 -------- .../Av1InverseTransformer.Dct4Operator.cs | 118 - .../Av1InverseTransformer.Dct64Operator.cs | 2278 ----------------- .../Av1InverseTransformer.Dct8Operator.cs | 238 -- ...v1InverseTransformer.Identity16Operator.cs | 87 - ...v1InverseTransformer.Identity32Operator.cs | 71 - ...Av1InverseTransformer.Identity4Operator.cs | 87 - ...Av1InverseTransformer.Identity8Operator.cs | 71 - .../Av1InverseTransformer.OutputOperator.cs | 100 - .../Av1/Transform/Av1InverseTransformer.cs | 2 +- .../Transform/Av1InverseTransformerFactory.cs | 4 +- .../Av1InverseWalshHadamardTransformer.cs | 12 +- .../Heif/Av1/Transform/Av1Transform1dMath.cs | 5 + .../Forward/Av1Adst16Forward1dOperator.cs | 21 + .../Forward/Av1Adst4Forward1dOperator.cs | 21 + .../Forward/Av1Adst8Forward1dOperator.cs | 21 + .../Forward/Av1Dct16Forward1dOperator.cs | 21 + .../Forward/Av1Dct32Forward1dOperator.cs | 21 + .../Forward/Av1Dct4Forward1dOperator.cs | 21 + .../Forward/Av1Dct64Forward1dOperator.cs | 21 + .../Forward/Av1Dct8Forward1dOperator.cs | 21 + .../Forward/Av1Identity16Forward1dOperator.cs | 21 + .../Forward/Av1Identity32Forward1dOperator.cs | 21 + .../Forward/Av1Identity4Forward1dOperator.cs | 21 + .../Forward/Av1Identity8Forward1dOperator.cs | 21 + .../Forward/IAv1ForwardTransform1dOperator.cs | 34 + .../IAv1InverseTransformOutputOperator.cs | 46 + .../Av1/Transform/IAv1Transform1dOperator.cs | 59 + .../Inverse/Av1Adst16Inverse1dOperator.cs | 568 ++++ .../Inverse/Av1Adst4Inverse1dOperator.cs | 168 ++ .../Inverse/Av1Adst8Inverse1dOperator.cs | 289 +++ .../Inverse/Av1Dct16Inverse1dOperator.cs | 475 ++++ .../Inverse/Av1Dct32Inverse1dOperator.cs | 1027 ++++++++ .../Inverse/Av1Dct4Inverse1dOperator.cs | 112 + .../Inverse/Av1Dct64Inverse1dOperator.cs | 2272 ++++++++++++++++ .../Inverse/Av1Dct8Inverse1dOperator.cs | 232 ++ .../Inverse/Av1Identity16Inverse1dOperator.cs | 81 + .../Inverse/Av1Identity32Inverse1dOperator.cs | 65 + .../Inverse/Av1Identity4Inverse1dOperator.cs | 81 + .../Inverse/Av1Identity8Inverse1dOperator.cs | 65 + .../Heif/Av1/Av1ForwardTransformTests.cs | 111 +- .../Heif/Av1/Av1InverseTransformTests.cs | 140 +- 62 files changed, 6124 insertions(+), 6278 deletions(-) delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs rename src/ImageSharp/Formats/Heif/Av1/Transform/{Av1ForwardTransformer.Operator.cs => Av1ForwardTransformer.cs} (91%) rename src/ImageSharp/Formats/Heif/Av1/Transform/{Av1InverseTransformer.Operator.cs => Av1Inverse2dTransformer.cs} (82%) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst16Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst4Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst8Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct16Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct32Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct4Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct64Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct8Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity16Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity32Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity4Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity8Operator.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.OutputOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 024c3802d..223486a48 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -856,7 +856,7 @@ No valid HEVC or AV1 color, compression, or bit-depth row may remain `unsupporte - Keep changes vertical and reviewable. A slice should add one behavior, its focused tests, independent evidence, and any required notice update. - Keep every AV1 prediction family on the established JPEG color-converter operator architecture. Each distinct traversal contract owns a family-named predictor type; its `.Operator.cs` defines the static interface, and its family-named files own the closed generic widest-to-narrowest SIMD traversal. Semantic `readonly struct` operators implement scalar, `Vector128`, `Vector256`, and `Vector512` arithmetic through that contract. Only modes which share the same traversal and contract may share a predictor family; do not nest a separate predictor beneath a broad intra/inter family or create hardware-width-specific class hierarchies. -- Keep AV1 forward and inverse transform operators under their family-named `Av1ForwardTransformer` and `Av1InverseTransformer` owners, matching the JPEG color-converter operator architecture. Their `.Operator.cs` files own the static contracts and generic dispatch, while the family-named operator files own each DCT, ADST, identity, and reconstruction-output implementation. Do not nest one-dimensional transform operators beneath a two-dimensional helper or name operator files after an implementation-detail dispatcher. +- Keep AV1 one-dimensional transform implementations as top-level semantic `readonly struct` operators. Forward operators belong to `Transform.Forward`, inverse operators belong to `Transform.Inverse`, and each family implements its top-level static-generic contract. Keep `Av1ForwardTransformer` and `Av1Inverse2dTransformer` as separate two-dimensional dispatchers, and keep `Av1InverseTransformOutputOperator` top-level. Do not wrap or nest transform operators beneath either dispatcher and do not name operator files after an implementation-detail owner. - Design SIMD-suitable codec work SIMD-first. Establish vector-friendly storage, operator boundaries, scratch ownership, traversal, every applicable lane width, and benchmark-gated dispatch before implementing the equivalent scalar fallback; never build a scalar production architecture and bolt SIMD onto it later. - Inspect every owning method and upstream invariant before adding guards. Validate external file data at the parser/model boundary and rely on those established invariants internally. - Do not extract one-use helpers merely to label code. Extract shared primitives only when they have genuine reuse or remove substantial complexity. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs deleted file mode 100644 index 10516e3d7..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst16Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixteen-point asymmetric discrete sine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator. - /// - internal readonly struct Adst16Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs deleted file mode 100644 index f0cd6ff74..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst4Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the four-point asymmetric discrete sine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the four-point AV1 forward asymmetric discrete sine transform operator. - /// - internal readonly struct Adst4Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs deleted file mode 100644 index 0df4c4414..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Adst8Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the eight-point asymmetric discrete sine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the eight-point AV1 forward asymmetric discrete sine transform operator. - /// - internal readonly struct Adst8Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs deleted file mode 100644 index 3039ae930..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct16Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixteen-point discrete cosine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the sixteen-point AV1 forward discrete cosine transform operator. - /// - internal readonly struct Dct16Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs deleted file mode 100644 index e662e6e08..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct32Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the thirty-two-point discrete cosine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the thirty-two-point AV1 forward discrete cosine transform operator. - /// - internal readonly struct Dct32Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs deleted file mode 100644 index 85eee7142..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct4Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the four-point discrete cosine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the four-point AV1 forward discrete cosine transform operator. - /// - internal readonly struct Dct4Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs deleted file mode 100644 index 8446f78a9..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct64Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixty-four-point discrete cosine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the sixty-four-point AV1 forward discrete cosine transform operator. - /// - internal readonly struct Dct64Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs deleted file mode 100644 index c4c4acc7a..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Dct8Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the eight-point discrete cosine forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the eight-point AV1 forward discrete cosine transform operator. - /// - internal readonly struct Dct8Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs deleted file mode 100644 index 61d85e1a9..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity16Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixteen-point identity forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the sixteen-point AV1 forward identity transform operator. - /// - internal readonly struct Identity16Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs deleted file mode 100644 index 7a65eaa01..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity32Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the thirty-two-point identity forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the thirty-two-point AV1 forward identity transform operator. - /// - internal readonly struct Identity32Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs deleted file mode 100644 index f112053cf..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity4Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the four-point identity forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the four-point AV1 forward identity transform operator. - /// - internal readonly struct Identity4Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs deleted file mode 100644 index 24edb99e4..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Identity8Operator.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the eight-point identity forward transform operator. -/// -internal static partial class Av1ForwardTransformer -{ - /// - /// Defines the eight-point AV1 forward identity transform operator. - /// - internal readonly struct Identity8Operator : IAv1ForwardTransform1dOperator - { - /// - public static void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct - => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs similarity index 91% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Operator.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs index f6b7ca07a..650d6a21b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs @@ -18,38 +18,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// Eight-bit blocks use saturating 16-bit stages where their normative ranges permit it; high-bit-depth and scalar /// fallback paths retain 32-bit stages. Both representations produce the same row-major coefficient contract. /// -internal static partial class Av1ForwardTransformer +internal static class Av1ForwardTransformer { - /// - /// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type. - /// - /// - /// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type - /// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch - /// inside the transform stages. - /// - internal interface IAv1ForwardTransform1dOperator - { - /// - /// Transforms the independent axes stored in each value lane. - /// - /// The scalar or SIMD value containing the independent transform axes. - /// The first value in the strided transform block. - /// The byte distance between consecutive input positions. - /// The byte distance between consecutive output positions. - /// The first fixed transform-stage buffer. - /// The second fixed transform-stage buffer. - /// The fixed-point precision of the cosine constants. - public static abstract void Transform( - ref byte values, - nint inputStride, - nint outputStride, - ref Av1TransformVector buffer0, - ref Av1TransformVector buffer1, - int cosBit) - where TValue : struct; - } - /// /// Resolves and applies the configured two-dimensional AV1 forward transform. /// @@ -95,40 +65,40 @@ internal static partial class Av1ForwardTransformer switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); + DispatchRow(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeColumn} column transform is not valid for {config.TransformSize}."); @@ -157,40 +127,40 @@ internal static partial class Av1ForwardTransformer switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct32: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Dct64: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Adst16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity4: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity8: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity16: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; case Av1TransformFunctionType.Identity32: - Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); + Transform2d(input, coefficients, stride, bitDepth, ref config, workspace); break; default: throw new InvalidImageContentException($"The {config.TransformFunctionTypeRow} row transform is not valid for {config.TransformSize}."); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs similarity index 82% rename from src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs rename to src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs index 5ee2f1249..23ddc7e2f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -16,106 +17,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// cross-lane permutations. Reconstruction adds the final residuals to their matching prediction lanes before /// narrowing to the decoded sample depth. /// -internal static partial class Av1InverseTransformer +internal static class Av1Inverse2dTransformer { - /// - /// The signed stage width whose fixed-point terminal operations require widened SIMD intermediates. - /// - private const byte WidenedIntermediateBitCount = 20; - - /// - /// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional inverse transform. - /// - /// - /// Each overload performs the same staged fixed-point inverse transform. In the SIMD overloads, each vector field identifies - /// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore - /// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage - /// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the - /// JIT to specialize every static interface call outside the stage network. - /// - internal interface IAv1InverseTransform1dOperator - { - /// - /// Transforms one axis when hardware vectorization is unavailable. - /// - /// The source values for the transform axis. - /// The destination values for the transform axis. - /// The fixed stage storage for the transform axis. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange); - - /// - /// Transforms four independent axes in parallel. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed stage storage for four transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); - - /// - /// Transforms eight independent axes in parallel. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed stage storage for eight transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static abstract void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange); - } - - /// - /// Defines how inverse-transform residuals are added to a decoded sample representation. - /// - /// - /// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions, - /// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable - /// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize. - /// - /// The decoded sample storage type. - internal interface IAv1InverseTransformOutputOperator - where TSample : unmanaged - { - /// - /// Adds one residual to a predicted sample and clips the result to the coded bit depth. - /// - /// The predicted sample. - /// The inverse-transform residual. - /// The coded sample bit depth. - /// The reconstructed sample. - public static abstract TSample Add(TSample prediction, int residual, int bitDepth); - - /// - /// Adds four residuals to four predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The four inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth); - - /// - /// Adds eight residuals to eight predicted samples and stores the clipped results. - /// - /// The first predicted sample. - /// The first destination sample. - /// The eight inverse-transform residuals. - /// The coded sample bit depth. - public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); - } - /// /// Applies an inverse transform and adds its residual to high-bit-depth predicted samples. /// @@ -136,7 +39,7 @@ internal static partial class Av1InverseTransformer ref Av1Transform2dFlipConfiguration config, Span workspace, int bitDepth) - => Transform2dAdd>( + => Transform2dAdd>( input, outputForRead, strideForRead, @@ -164,7 +67,7 @@ internal static partial class Av1InverseTransformer int strideForWrite, ref Av1Transform2dFlipConfiguration config, Span workspace) - => Transform2dAdd>( + => Transform2dAdd>( input, outputForRead, strideForRead, @@ -193,62 +96,62 @@ internal static partial class Av1InverseTransformer switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct32: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct64: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity4: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity8: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity16: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity32: - DispatchRow( + DispatchRow( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; @@ -271,67 +174,67 @@ internal static partial class Av1InverseTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct32: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Dct64: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Adst16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity4: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity8: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity16: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; case Av1TransformFunctionType.Identity32: - Transform2d( + Transform2d( input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); break; @@ -354,8 +257,8 @@ internal static partial class Av1InverseTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1InverseTransform1dOperator - where TRowOperator : struct, IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); @@ -406,8 +309,8 @@ internal static partial class Av1InverseTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1InverseTransform1dOperator - where TRowOperator : struct, IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { const int laneCount = 8; const int vectorLength = Av1Constants.MaxTransformSize * laneCount; @@ -562,8 +465,8 @@ internal static partial class Av1InverseTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1InverseTransform1dOperator - where TRowOperator : struct, IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { const int laneCount = 4; const int vectorLength = Av1Constants.MaxTransformSize * laneCount; @@ -694,8 +597,8 @@ internal static partial class Av1InverseTransformer int bitDepth) where TSample : unmanaged where TOutputOperator : struct, IAv1InverseTransformOutputOperator - where TColumnOperator : struct, IAv1InverseTransform1dOperator - where TRowOperator : struct, IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs new file mode 100644 index 000000000..08d035cf8 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs @@ -0,0 +1,94 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Reconstructs AV1 samples from predicted values and inverse-transform residuals. +/// +/// +/// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to +/// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width +/// stores. The closed specialization removes storage-type branches from hot loops. +/// +/// The decoded sample storage type. +internal readonly struct Av1InverseTransformOutputOperator : IAv1InverseTransformOutputOperator + where TSample : unmanaged +{ + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TSample Add(TSample prediction, int residual, int bitDepth) + { + // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each + // closed transform so storage selection does not introduce a branch in the reconstruction loop. + if (typeof(TSample) == typeof(byte)) + { + byte value = (byte)Math.Clamp(Unsafe.As(ref prediction) + residual, byte.MinValue, byte.MaxValue); + return Unsafe.As(ref value); + } + + short result = (short)Math.Clamp(Unsafe.As(ref prediction) + residual, 0, (1 << bitDepth) - 1); + return Unsafe.As(ref result); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth) + { + if (typeof(TSample) == typeof(byte)) + { + // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach + // memory, which keeps reconstruction valid at a tightly packed row boundary. + ref byte source = ref Unsafe.As(ref prediction); + uint packed = Unsafe.ReadUnaligned(ref source); + Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32(); + Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue)); + Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt32().ToScalar()); + return; + } + + ref short highBitDepthSource = ref Unsafe.As(ref prediction); + ulong highBitDepthPacked = Unsafe.ReadUnaligned(ref Unsafe.As(ref highBitDepthSource)); + Vector128 highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16()); + Vector128 highBitDepthReconstructed = + Vector128.Clamp(highBitDepthPredicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1)); + + Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar()); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth) + { + if (typeof(TSample) == typeof(byte)) + { + // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store + // covers only those reconstructed samples and does not require destination padding. + ref byte source = ref Unsafe.As(ref prediction); + ulong packed = Unsafe.ReadUnaligned(ref source); + Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32(); + Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue)); + Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt64().ToScalar()); + return; + } + + ref short highBitDepthSource = ref Unsafe.As(ref prediction); + Vector256 highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource)); + Vector256 highBitDepthReconstructed = + Vector256.Clamp(highBitDepthPredicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1)); + + Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper()); + narrowed.StoreUnsafe(ref Unsafe.As(ref destination)); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst16Operator.cs deleted file mode 100644 index 5cdcc1c47..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst16Operator.cs +++ /dev/null @@ -1,574 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixteen-point asymmetric discrete sine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Adst16Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen spatial-domain residual values. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[15]; - output[1] = input[0]; - output[2] = input[13]; - output[3] = input[2]; - output[4] = input[11]; - output[5] = input[4]; - output[6] = input[9]; - output[7] = input[6]; - output[8] = input[7]; - output[9] = input[8]; - output[10] = input[5]; - output[11] = input[10]; - output[12] = input[3]; - output[13] = input[12]; - output[14] = input[1]; - output[15] = input[14]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); - step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); - step[12] = output[12]; - step[13] = output[13]; - step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[8]; - output[2] = step[12]; - output[3] = -step[4]; - output[4] = step[6]; - output[5] = -step[14]; - output[6] = step[10]; - output[7] = -step[2]; - output[8] = step[3]; - output[9] = -step[11]; - output[10] = step[15]; - output[11] = -step[7]; - output[12] = step[5]; - output[13] = -step[13]; - output[14] = step[9]; - output[15] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V15; - output.V1 = input.V0; - output.V2 = input.V13; - output.V3 = input.V2; - output.V4 = input.V11; - output.V5 = input.V4; - output.V6 = input.V9; - output.V7 = input.V6; - output.V8 = input.V7; - output.V9 = input.V8; - output.V10 = input.V5; - output.V11 = input.V10; - output.V12 = input.V3; - output.V13 = input.V12; - output.V14 = input.V1; - output.V15 = input.V14; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V8; - output.V2 = step.V12; - output.V3 = -step.V4; - output.V4 = step.V6; - output.V5 = -step.V14; - output.V6 = step.V10; - output.V7 = -step.V2; - output.V8 = step.V3; - output.V9 = -step.V11; - output.V10 = step.V15; - output.V11 = -step.V7; - output.V12 = step.V5; - output.V13 = -step.V13; - output.V14 = step.V9; - output.V15 = -step.V1; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V15; - output.V1 = input.V0; - output.V2 = input.V13; - output.V3 = input.V2; - output.V4 = input.V11; - output.V5 = input.V4; - output.V6 = input.V9; - output.V7 = input.V6; - output.V8 = input.V7; - output.V9 = input.V8; - output.V10 = input.V5; - output.V11 = input.V10; - output.V12 = input.V3; - output.V13 = input.V12; - output.V14 = input.V1; - output.V15 = input.V14; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); - - // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); - - // Stage 4 reverses the pi/16 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); - - // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); - - // Stage 6 reverses the pi/8 and 3pi/8 rotations. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); - - // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); - - // Stage 8 reverses the pi/4 rotations for the middle pairs. - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); - - // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V8; - output.V2 = step.V12; - output.V3 = -step.V4; - output.V4 = step.V6; - output.V5 = -step.V14; - output.V6 = step.V10; - output.V7 = -step.V2; - output.V8 = step.V3; - output.V9 = -step.V11; - output.V10 = step.V15; - output.V11 = -step.V7; - output.V12 = step.V5; - output.V13 = -step.V13; - output.V14 = step.V9; - output.V15 = -step.V1; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst4Operator.cs deleted file mode 100644 index 50e1db2de..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst4Operator.cs +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the four-point asymmetric discrete sine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the four-point AV1 inverse asymmetric discrete sine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Adst4Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform. - /// - /// The four frequency-domain coefficients. - /// The four spatial-domain residual values. - /// The stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the sine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - - // libaom widens the complete four-point factorization because the products retain their fixed-point scale - // until the final shift. The stage buffer is therefore unnecessary for this transform size. - long x0 = input[0]; - long x1 = input[1]; - long x2 = input[2]; - long x3 = input[3]; - - _ = step; - _ = stageRange; - - // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel. - if ((x0 | x1 | x2 | x3) == 0) - { - output[..4].Clear(); - return; - } - - // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3. - long s0 = sinpi[1] * x0; - long s1 = sinpi[2] * x0; - long s2 = sinpi[3] * x1; - long s3 = sinpi[4] * x2; - long s4 = sinpi[1] * x2; - long s5 = sinpi[2] * x3; - long s6 = sinpi[4] * x3; - long s7 = (x0 - x2) + x3; - - // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding. - s0 += s3; - s1 -= s4; - s3 = s2; - s2 = sinpi[3] * s7; - s0 += s5; - s1 -= s6; - x0 = s0 + s3; - x1 = s1 + s3; - x2 = s2; - x3 = (s0 + s1) - s3; - - output[0] = Av1Math.RoundShift(x0, cosBit); - output[1] = Av1Math.RoundShift(x1, cosBit); - output[2] = Av1Math.RoundShift(x2, cosBit); - output[3] = Av1Math.RoundShift(x3, cosBit); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit, stageRange[0] >= WidenedIntermediateBitCount); - _ = step; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - TransformCore(ref input, ref output, cosBit, stageRange[0] >= WidenedIntermediateBitCount); - _ = step; - } - - /// - /// Applies the inverse four-point matrix to four independent axes. - /// - /// The source values for four transform axes. - /// The destination values for four transform axes. - /// The fixed-point precision of the sine constants. - /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. - private static void TransformCore( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - int cosBit, - bool widenedRound) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector128 x0 = input.V0; - Vector128 x1 = input.V1; - Vector128 x2 = input.V2; - Vector128 x3 = input.V3; - - // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row - // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow - // a valid Int32 fixed-point sum. - if (widenedRound) - { - output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - return; - } - - output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - - /// - /// Applies the inverse four-point matrix to eight independent axes. - /// - /// The source values for eight transform axes. - /// The destination values for eight transform axes. - /// The fixed-point precision of the sine constants. - /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. - private static void TransformCore( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - int cosBit, - bool widenedRound) - { - ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); - Vector256 x0 = input.V0; - Vector256 x1 = input.V1; - Vector256 x2 = input.V2; - Vector256 x3 = input.V3; - - if (widenedRound) - { - output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - return; - } - - output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); - output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); - output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); - output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst8Operator.cs deleted file mode 100644 index 5340275bf..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Adst8Operator.cs +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the eight-point asymmetric discrete sine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Adst8Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform. - /// - /// The eight frequency-domain coefficients. - /// The eight spatial-domain residual values. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output[0] = input[7]; - output[1] = input[0]; - output[2] = input[5]; - output[3] = input[2]; - output[4] = input[3]; - output[5] = input[4]; - output[6] = input[1]; - output[7] = input[6]; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); - step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); - step[4] = output[4]; - step[5] = output[5]; - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output[0] = step[0]; - output[1] = -step[4]; - output[2] = step[6]; - output[3] = -step[2]; - output[4] = step[3]; - output[5] = -step[7]; - output[6] = step[5]; - output[7] = -step[1]; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V7; - output.V1 = input.V0; - output.V2 = input.V5; - output.V3 = input.V2; - output.V4 = input.V3; - output.V5 = input.V4; - output.V6 = input.V1; - output.V7 = input.V6; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V4; - output.V2 = step.V6; - output.V3 = -step.V2; - output.V4 = step.V3; - output.V5 = -step.V7; - output.V6 = step.V5; - output.V7 = -step.V1; - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. - stage++; - output.V0 = input.V7; - output.V1 = input.V0; - output.V2 = input.V5; - output.V3 = input.V2; - output.V4 = input.V3; - output.V5 = input.V4; - output.V6 = input.V1; - output.V7 = input.V6; - - // Stage 2 applies the terminal odd-angle rotations in reverse. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); - - // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); - - // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); - - // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); - - // Stage 6 reverses the pi/4 rotations for the middle pairs. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); - - // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. - output.V0 = step.V0; - output.V1 = -step.V4; - output.V2 = step.V6; - output.V3 = -step.V2; - output.V4 = step.V3; - output.V5 = -step.V7; - output.V6 = step.V5; - output.V7 = -step.V1; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct16Operator.cs deleted file mode 100644 index 32aa7525a..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct16Operator.cs +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixteen-point discrete cosine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the 16-point AV1 inverse discrete cosine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Dct16Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative 16-point AV1 inverse discrete cosine transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen spatial-domain residual values. - /// The sixteen-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[8]; - output[2] = input[4]; - output[3] = input[12]; - output[4] = input[2]; - output[5] = input[10]; - output[6] = input[6]; - output[7] = input[14]; - output[8] = input[1]; - output[9] = input[9]; - output[10] = input[5]; - output[11] = input[13]; - output[12] = input[3]; - output[13] = input[11]; - output[14] = input[7]; - output[15] = input[15]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V8; - output.V2 = input.V4; - output.V3 = input.V12; - output.V4 = input.V2; - output.V5 = input.V10; - output.V6 = input.V6; - output.V7 = input.V14; - output.V8 = input.V1; - output.V9 = input.V9; - output.V10 = input.V5; - output.V11 = input.V13; - output.V12 = input.V3; - output.V13 = input.V11; - output.V14 = input.V7; - output.V15 = input.V15; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V8; - output.V2 = input.V4; - output.V3 = input.V12; - output.V4 = input.V2; - output.V5 = input.V10; - output.V6 = input.V6; - output.V7 = input.V14; - output.V8 = input.V1; - output.V9 = input.V9; - output.V10 = input.V5; - output.V11 = input.V13; - output.V12 = input.V3; - output.V13 = input.V11; - output.V14 = input.V7; - output.V15 = input.V15; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - - // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); - - // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - - // Stage 5 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); - - // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - - // Stage 7 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct32Operator.cs deleted file mode 100644 index 798af9fa7..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct32Operator.cs +++ /dev/null @@ -1,1033 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the thirty-two-point discrete cosine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the 32-point AV1 inverse discrete cosine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Dct32Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative 32-point AV1 inverse discrete cosine transform. - /// - /// The 32 frequency-domain coefficients. - /// The 32 spatial-domain residual values. - /// The 32-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[16]; - output[2] = input[8]; - output[3] = input[24]; - output[4] = input[4]; - output[5] = input[20]; - output[6] = input[12]; - output[7] = input[28]; - output[8] = input[2]; - output[9] = input[18]; - output[10] = input[10]; - output[11] = input[26]; - output[12] = input[6]; - output[13] = input[22]; - output[14] = input[14]; - output[15] = input[30]; - output[16] = input[1]; - output[17] = input[17]; - output[18] = input[9]; - output[19] = input[25]; - output[20] = input[5]; - output[21] = input[21]; - output[22] = input[13]; - output[23] = input[29]; - output[24] = input[3]; - output[25] = input[19]; - output[26] = input[11]; - output[27] = input[27]; - output[28] = input[7]; - output[29] = input[23]; - output[30] = input[15]; - output[31] = input[31]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); - step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); - step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); - output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); - output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); - output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); - output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); - output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); - output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); - output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); - output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); - output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); - output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); - step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); - step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); - step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); - step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); - step[16] = output[16]; - step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); - step[19] = output[19]; - step[20] = output[20]; - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); - step[23] = output[23]; - step[24] = output[24]; - step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); - step[27] = output[27]; - step[28] = output[28]; - step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); - step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); - step[31] = output[31]; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - output[8] = step[8]; - output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); - output[11] = step[11]; - output[12] = step[12]; - output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); - output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); - output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); - output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); - output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); - output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); - output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); - output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); - step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); - step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); - step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); - step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); - step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); - step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); - step[30] = output[30]; - step[31] = output[31]; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - output[8] = step[8]; - output[9] = step[9]; - output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); - output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); - output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); - output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); - output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); - output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); - output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); - output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); - output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); - step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); - step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); - step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); - step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); - step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); - step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); - step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); - step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); - step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); - step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); - step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); - step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); - step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); - step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); - step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); - output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); - output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); - output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); - output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); - output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); - output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); - output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); - output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); - output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); - output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); - output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); - output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); - output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); - output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); - output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); - output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V16; - output.V2 = input.V8; - output.V3 = input.V24; - output.V4 = input.V4; - output.V5 = input.V20; - output.V6 = input.V12; - output.V7 = input.V28; - output.V8 = input.V2; - output.V9 = input.V18; - output.V10 = input.V10; - output.V11 = input.V26; - output.V12 = input.V6; - output.V13 = input.V22; - output.V14 = input.V14; - output.V15 = input.V30; - output.V16 = input.V1; - output.V17 = input.V17; - output.V18 = input.V9; - output.V19 = input.V25; - output.V20 = input.V5; - output.V21 = input.V21; - output.V22 = input.V13; - output.V23 = input.V29; - output.V24 = input.V3; - output.V25 = input.V19; - output.V26 = input.V11; - output.V27 = input.V27; - output.V28 = input.V7; - output.V29 = input.V23; - output.V30 = input.V15; - output.V31 = input.V31; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); - step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); - step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); - output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); - output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); - output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); - output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); - output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); - output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); - output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); - output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); - output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); - output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); - step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); - step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); - step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); - step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); - step.V16 = output.V16; - step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); - step.V31 = output.V31; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - output.V8 = step.V8; - output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); - output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); - output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); - output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); - output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); - output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); - output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); - step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); - step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); - step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); - output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); - output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); - output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); - output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); - step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); - step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); - step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); - output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); - output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); - output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V16; - output.V2 = input.V8; - output.V3 = input.V24; - output.V4 = input.V4; - output.V5 = input.V20; - output.V6 = input.V12; - output.V7 = input.V28; - output.V8 = input.V2; - output.V9 = input.V18; - output.V10 = input.V10; - output.V11 = input.V26; - output.V12 = input.V6; - output.V13 = input.V22; - output.V14 = input.V14; - output.V15 = input.V30; - output.V16 = input.V1; - output.V17 = input.V17; - output.V18 = input.V9; - output.V19 = input.V25; - output.V20 = input.V5; - output.V21 = input.V21; - output.V22 = input.V13; - output.V23 = input.V29; - output.V24 = input.V3; - output.V25 = input.V19; - output.V26 = input.V11; - output.V27 = input.V27; - output.V28 = input.V7; - output.V29 = input.V23; - output.V30 = input.V15; - output.V31 = input.V31; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); - step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); - step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); - output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); - output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); - output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); - output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); - output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); - output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); - output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); - output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); - output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); - output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - range = stageRange[stage]; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); - step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); - step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); - step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); - step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); - step.V16 = output.V16; - step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); - step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); - step.V31 = output.V31; - - // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - output.V8 = step.V8; - output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); - output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); - output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); - output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); - output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); - output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); - output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); - output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); - - // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); - step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); - step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); - step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); - step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); - step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); - step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 7 widens the reconstructed groups through their next butterfly level. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); - output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); - output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); - output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); - output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); - output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); - output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); - - // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - range = stageRange[stage]; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); - step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); - step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); - step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); - step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); - step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); - step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); - step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); - step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); - step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); - step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); - step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - - // Stage 9 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); - output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); - output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); - output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); - output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); - output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); - output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); - output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); - output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); - output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); - output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); - output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); - output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); - output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); - output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); - output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); - output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct4Operator.cs deleted file mode 100644 index cc5f71931..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct4Operator.cs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the four-point discrete cosine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the four-point AV1 inverse discrete cosine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Dct4Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative four-point AV1 inverse discrete cosine transform. - /// - /// The four frequency-domain coefficients. - /// The four spatial-domain residual values. - /// The four-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output[0] = input[0]; - output[1] = input[2]; - output[2] = input[1]; - output[3] = input[3]; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output.V0 = input.V0; - output.V1 = input.V2; - output.V2 = input.V1; - output.V3 = input.V3; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. - output.V0 = input.V0; - output.V1 = input.V2; - output.V2 = input.V1; - output.V3 = input.V3; - - // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - - // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. - byte range = stageRange[3]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct64Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct64Operator.cs deleted file mode 100644 index 749d85ec3..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct64Operator.cs +++ /dev/null @@ -1,2278 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixty-four-point discrete cosine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the 64-point AV1 inverse discrete cosine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Dct64Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative 64-point AV1 inverse discrete cosine transform. - /// - /// The 64 frequency-domain coefficients. - /// The 64 spatial-domain residual values. - /// The 64-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[32]; - output[2] = input[16]; - output[3] = input[48]; - output[4] = input[8]; - output[5] = input[40]; - output[6] = input[24]; - output[7] = input[56]; - output[8] = input[4]; - output[9] = input[36]; - output[10] = input[20]; - output[11] = input[52]; - output[12] = input[12]; - output[13] = input[44]; - output[14] = input[28]; - output[15] = input[60]; - output[16] = input[2]; - output[17] = input[34]; - output[18] = input[18]; - output[19] = input[50]; - output[20] = input[10]; - output[21] = input[42]; - output[22] = input[26]; - output[23] = input[58]; - output[24] = input[6]; - output[25] = input[38]; - output[26] = input[22]; - output[27] = input[54]; - output[28] = input[14]; - output[29] = input[46]; - output[30] = input[30]; - output[31] = input[62]; - output[32] = input[1]; - output[33] = input[33]; - output[34] = input[17]; - output[35] = input[49]; - output[36] = input[9]; - output[37] = input[41]; - output[38] = input[25]; - output[39] = input[57]; - output[40] = input[5]; - output[41] = input[37]; - output[42] = input[21]; - output[43] = input[53]; - output[44] = input[13]; - output[45] = input[45]; - output[46] = input[29]; - output[47] = input[61]; - output[48] = input[3]; - output[49] = input[35]; - output[50] = input[19]; - output[51] = input[51]; - output[52] = input[11]; - output[53] = input[43]; - output[54] = input[27]; - output[55] = input[59]; - output[56] = input[7]; - output[57] = input[39]; - output[58] = input[23]; - output[59] = input[55]; - output[60] = input[15]; - output[61] = input[47]; - output[62] = input[31]; - output[63] = input[63]; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); - step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); - step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); - output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); - output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); - output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); - step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); - step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); - step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); - step[35] = output[35]; - step[36] = output[36]; - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); - step[39] = output[39]; - step[40] = output[40]; - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); - step[43] = output[43]; - step[44] = output[44]; - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); - step[47] = output[47]; - step[48] = output[48]; - step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); - step[51] = output[51]; - step[52] = output[52]; - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); - step[55] = output[55]; - step[56] = output[56]; - step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); - step[59] = output[59]; - step[60] = output[60]; - step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); - step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); - step[63] = output[63]; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); - output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); - output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); - output[19] = step[19]; - output[20] = step[20]; - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); - output[23] = step[23]; - output[24] = step[24]; - output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); - output[27] = step[27]; - output[28] = step[28]; - output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); - output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); - step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); - step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); - step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); - step[11] = output[11]; - step[12] = output[12]; - step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); - step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); - step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); - step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); - step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); - output[4] = step[4]; - output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); - output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); - output[7] = step[7]; - output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); - output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); - output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); - output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); - step[8] = output[8]; - step[9] = output[9]; - step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); - step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); - step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); - step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); - step[14] = output[14]; - step[15] = output[15]; - step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); - step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); - step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); - step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); - step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); - step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); - step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); - step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); - output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); - output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); - output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); - output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); - output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); - output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); - output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); - step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); - step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); - step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); - step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); - step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); - step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); - step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); - step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); - step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); - step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); - step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); - step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); - step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); - step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); - step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); - step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); - step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); - step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); - step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); - step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); - step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); - step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); - step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); - step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); - step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); - step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); - step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); - step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); - step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); - step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); - step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); - step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); - step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); - step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); - step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); - step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); - step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); - step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); - step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); - step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); - step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); - step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); - step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); - step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); - step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); - output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); - output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); - output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); - output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); - output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); - output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); - output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); - output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); - output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); - output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); - output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); - output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); - output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); - output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); - output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); - output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); - output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); - output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); - output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); - output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); - output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); - output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); - output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); - output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); - output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); - output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); - output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); - output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); - output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); - output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); - output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); - output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); - output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); - output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); - output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); - output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); - output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); - output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); - output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); - output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); - output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); - output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); - output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); - output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); - output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); - output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); - output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); - output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); - output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); - output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); - output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); - output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); - output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); - output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); - output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); - output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); - output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); - output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); - output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V32; - output.V2 = input.V16; - output.V3 = input.V48; - output.V4 = input.V8; - output.V5 = input.V40; - output.V6 = input.V24; - output.V7 = input.V56; - output.V8 = input.V4; - output.V9 = input.V36; - output.V10 = input.V20; - output.V11 = input.V52; - output.V12 = input.V12; - output.V13 = input.V44; - output.V14 = input.V28; - output.V15 = input.V60; - output.V16 = input.V2; - output.V17 = input.V34; - output.V18 = input.V18; - output.V19 = input.V50; - output.V20 = input.V10; - output.V21 = input.V42; - output.V22 = input.V26; - output.V23 = input.V58; - output.V24 = input.V6; - output.V25 = input.V38; - output.V26 = input.V22; - output.V27 = input.V54; - output.V28 = input.V14; - output.V29 = input.V46; - output.V30 = input.V30; - output.V31 = input.V62; - output.V32 = input.V1; - output.V33 = input.V33; - output.V34 = input.V17; - output.V35 = input.V49; - output.V36 = input.V9; - output.V37 = input.V41; - output.V38 = input.V25; - output.V39 = input.V57; - output.V40 = input.V5; - output.V41 = input.V37; - output.V42 = input.V21; - output.V43 = input.V53; - output.V44 = input.V13; - output.V45 = input.V45; - output.V46 = input.V29; - output.V47 = input.V61; - output.V48 = input.V3; - output.V49 = input.V35; - output.V50 = input.V19; - output.V51 = input.V51; - output.V52 = input.V11; - output.V53 = input.V43; - output.V54 = input.V27; - output.V55 = input.V59; - output.V56 = input.V7; - output.V57 = input.V39; - output.V58 = input.V23; - output.V59 = input.V55; - output.V60 = input.V15; - output.V61 = input.V47; - output.V62 = input.V31; - output.V63 = input.V63; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = output.V21; - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = output.V26; - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); - step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); - step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = step.V10; - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = step.V13; - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); - output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); - output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); - step.V43 = output.V43; - step.V44 = output.V44; - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); - step.V51 = output.V51; - step.V52 = output.V52; - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); - step.V63 = output.V63; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); - output.V19 = step.V19; - output.V20 = step.V20; - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); - output.V27 = step.V27; - output.V28 = step.V28; - output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = output.V41; - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); - step.V54 = output.V54; - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); - output.V22 = step.V22; - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = step.V25; - output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); - step.V44 = output.V44; - step.V45 = output.V45; - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = output.V50; - step.V51 = output.V51; - step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = step.V18; - output.V19 = step.V19; - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V28 = step.V28; - output.V29 = step.V29; - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); - step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); - step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); - step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); - step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); - step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); - step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = output.V37; - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = output.V58; - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); - output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); - output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); - output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); - output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); - output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); - output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); - output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); - output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); - output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V32; - output.V2 = input.V16; - output.V3 = input.V48; - output.V4 = input.V8; - output.V5 = input.V40; - output.V6 = input.V24; - output.V7 = input.V56; - output.V8 = input.V4; - output.V9 = input.V36; - output.V10 = input.V20; - output.V11 = input.V52; - output.V12 = input.V12; - output.V13 = input.V44; - output.V14 = input.V28; - output.V15 = input.V60; - output.V16 = input.V2; - output.V17 = input.V34; - output.V18 = input.V18; - output.V19 = input.V50; - output.V20 = input.V10; - output.V21 = input.V42; - output.V22 = input.V26; - output.V23 = input.V58; - output.V24 = input.V6; - output.V25 = input.V38; - output.V26 = input.V22; - output.V27 = input.V54; - output.V28 = input.V14; - output.V29 = input.V46; - output.V30 = input.V30; - output.V31 = input.V62; - output.V32 = input.V1; - output.V33 = input.V33; - output.V34 = input.V17; - output.V35 = input.V49; - output.V36 = input.V9; - output.V37 = input.V41; - output.V38 = input.V25; - output.V39 = input.V57; - output.V40 = input.V5; - output.V41 = input.V37; - output.V42 = input.V21; - output.V43 = input.V53; - output.V44 = input.V13; - output.V45 = input.V45; - output.V46 = input.V29; - output.V47 = input.V61; - output.V48 = input.V3; - output.V49 = input.V35; - output.V50 = input.V19; - output.V51 = input.V51; - output.V52 = input.V11; - output.V53 = input.V43; - output.V54 = input.V27; - output.V55 = input.V59; - output.V56 = input.V7; - output.V57 = input.V39; - output.V58 = input.V23; - output.V59 = input.V55; - output.V60 = input.V15; - output.V61 = input.V47; - output.V62 = input.V31; - output.V63 = input.V63; - - // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = output.V10; - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = output.V13; - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = output.V16; - step.V17 = output.V17; - step.V18 = output.V18; - step.V19 = output.V19; - step.V20 = output.V20; - step.V21 = output.V21; - step.V22 = output.V22; - step.V23 = output.V23; - step.V24 = output.V24; - step.V25 = output.V25; - step.V26 = output.V26; - step.V27 = output.V27; - step.V28 = output.V28; - step.V29 = output.V29; - step.V30 = output.V30; - step.V31 = output.V31; - step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); - step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); - step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); - - // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = step.V4; - output.V5 = step.V5; - output.V6 = step.V6; - output.V7 = step.V7; - output.V8 = step.V8; - output.V9 = step.V9; - output.V10 = step.V10; - output.V11 = step.V11; - output.V12 = step.V12; - output.V13 = step.V13; - output.V14 = step.V14; - output.V15 = step.V15; - output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); - output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); - output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); - - // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = output.V4; - step.V5 = output.V5; - step.V6 = output.V6; - step.V7 = output.V7; - step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); - step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); - step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); - step.V43 = output.V43; - step.V44 = output.V44; - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); - step.V51 = output.V51; - step.V52 = output.V52; - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); - step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); - step.V63 = output.V63; - - // Stage 5 widens the nested groups through the next butterfly level. - stage++; - output.V0 = step.V0; - output.V1 = step.V1; - output.V2 = step.V2; - output.V3 = step.V3; - output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); - output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); - output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); - output.V19 = step.V19; - output.V20 = step.V20; - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); - output.V27 = step.V27; - output.V28 = step.V28; - output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); - output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); - - // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. - stage++; - step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); - step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); - step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); - step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); - step.V11 = output.V11; - step.V12 = output.V12; - step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); - step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); - step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = output.V40; - step.V41 = output.V41; - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); - step.V54 = output.V54; - step.V55 = output.V55; - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); - step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); - step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); - output.V4 = step.V4; - output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); - output.V7 = step.V7; - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); - output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); - output.V22 = step.V22; - output.V23 = step.V23; - output.V24 = step.V24; - output.V25 = step.V25; - output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); - output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); - output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); - - // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); - step.V8 = output.V8; - step.V9 = output.V9; - step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); - step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); - step.V14 = output.V14; - step.V15 = output.V15; - step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); - step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); - step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); - step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); - step.V44 = output.V44; - step.V45 = output.V45; - step.V46 = output.V46; - step.V47 = output.V47; - step.V48 = output.V48; - step.V49 = output.V49; - step.V50 = output.V50; - step.V51 = output.V51; - step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); - step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); - step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); - step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); - step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 9 widens the reconstructed groups through their next butterfly level. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); - output.V16 = step.V16; - output.V17 = step.V17; - output.V18 = step.V18; - output.V19 = step.V19; - output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); - output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); - output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); - output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); - output.V28 = step.V28; - output.V29 = step.V29; - output.V30 = step.V30; - output.V31 = step.V31; - output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); - - // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); - step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); - step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); - step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); - step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); - step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); - step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); - step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); - step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); - step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); - step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); - step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); - step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); - step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); - step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); - step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); - step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); - step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); - step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); - step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); - step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); - step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); - step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); - step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); - step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); - step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); - step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); - step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); - step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); - step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); - step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); - step.V32 = output.V32; - step.V33 = output.V33; - step.V34 = output.V34; - step.V35 = output.V35; - step.V36 = output.V36; - step.V37 = output.V37; - step.V38 = output.V38; - step.V39 = output.V39; - step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); - step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); - step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); - step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); - step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); - step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); - step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); - step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); - step.V56 = output.V56; - step.V57 = output.V57; - step.V58 = output.V58; - step.V59 = output.V59; - step.V60 = output.V60; - step.V61 = output.V61; - step.V62 = output.V62; - step.V63 = output.V63; - - // Stage 11 merges the even and odd halves into spatial order and clamps every result. - stage++; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); - output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); - output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); - output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); - output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); - output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); - output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); - output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); - output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); - output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); - output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); - output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); - output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); - output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); - output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); - output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); - output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); - output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); - output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); - output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); - output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); - output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); - output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); - output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); - output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); - output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); - output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); - output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); - output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); - output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); - output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); - output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); - output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); - output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); - output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); - output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); - output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); - output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); - output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); - output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); - output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); - output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); - output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); - output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); - output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); - output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); - output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); - output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); - output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); - output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); - output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); - output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); - output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); - output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); - output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); - output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); - output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); - output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); - output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); - output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct8Operator.cs deleted file mode 100644 index 51602a17b..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Dct8Operator.cs +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the eight-point discrete cosine inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the eight-point AV1 inverse discrete cosine transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply - /// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. - /// - internal readonly struct Dct8Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative eight-point AV1 inverse discrete cosine transform. - /// - /// The eight frequency-domain coefficients. - /// The eight spatial-domain residual values. - /// The eight-element stage buffer owned by the containing two-dimensional transform. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output[0] = input[0]; - output[1] = input[4]; - output[2] = input[2]; - output[3] = input[6]; - output[4] = input[1]; - output[5] = input[5]; - output[6] = input[3]; - output[7] = input[7]; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); - step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); - step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); - output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); - output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); - output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); - output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); - output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); - step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); - step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); - step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); - step[4] = output[4]; - step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); - step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); - step[7] = output[7]; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); - output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); - output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); - output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); - output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); - output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); - output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); - output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V4; - output.V2 = input.V2; - output.V3 = input.V6; - output.V4 = input.V1; - output.V5 = input.V5; - output.V6 = input.V3; - output.V7 = input.V7; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - } - - /// - /// Applies the transform to four independent axes in parallel. - /// - /// The source values for the parallel transform axes. - /// The destination values for the parallel transform axes. - /// The fixed stage storage for the parallel transform axes. - /// The fixed-point precision of the cosine constants. - /// The signed-bit range assigned to each transform stage. - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - int stage = 0; - - // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. - stage++; - output.V0 = input.V0; - output.V1 = input.V4; - output.V2 = input.V2; - output.V3 = input.V6; - output.V4 = input.V1; - output.V5 = input.V5; - output.V6 = input.V3; - output.V7 = input.V7; - - // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. - stage++; - step.V0 = output.V0; - step.V1 = output.V1; - step.V2 = output.V2; - step.V3 = output.V3; - step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); - step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); - step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); - - // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. - stage++; - byte range = stageRange[stage]; - output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); - output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); - output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); - output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); - output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); - output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); - - // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. - stage++; - step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); - step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); - step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); - step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); - step.V4 = output.V4; - step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); - step.V7 = output.V7; - - // Stage 5 merges the even and odd halves into spatial order and clamps every result. - stage++; - range = stageRange[stage]; - output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); - output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); - output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); - output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); - output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); - output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); - output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); - output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity16Operator.cs deleted file mode 100644 index 0837861e9..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity16Operator.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sixteen-point identity inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the sixteen-point AV1 inverse identity transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, - /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. - /// - internal readonly struct Identity16Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative sixteen-point AV1 inverse identity transform. - /// - /// The sixteen frequency-domain coefficients. - /// The sixteen scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization. - for (int i = 0; i < 16; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and - // rounding sequence, matching libaom without changing the established lower-range SIMD path. - if (stageRange[0] >= WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - if (stageRange[0] >= WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity32Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity32Operator.cs deleted file mode 100644 index 331ebb283..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity32Operator.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the thirty-two-point identity inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the thirty-two-point AV1 inverse identity transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, - /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. - /// - internal readonly struct Identity32Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative thirty-two-point AV1 inverse identity transform. - /// - /// The thirty-two frequency-domain coefficients. - /// The thirty-two scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization. - for (int i = 0; i < 32; i++) - { - output[i] = input[i] * 4; - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity4Operator.cs deleted file mode 100644 index 59830776d..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity4Operator.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the four-point identity inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the four-point AV1 inverse identity transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, - /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. - /// - internal readonly struct Identity4Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative four-point AV1 inverse identity transform. - /// - /// The four frequency-domain coefficients. - /// The four scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization. - for (int i = 0; i < 4; i++) - { - output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product. - // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges. - if (stageRange[0] >= WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - if (stageRange[0] >= WidenedIntermediateBitCount) - { - Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - else - { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); - } - - _ = step; - _ = cosBit; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity8Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity8Operator.cs deleted file mode 100644 index 66914e53f..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.Identity8Operator.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.Intrinsics; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the eight-point identity inverse transform operator. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Defines the eight-point AV1 inverse identity transform operator. - /// - /// - /// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, - /// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. - /// - internal readonly struct Identity8Operator : IAv1InverseTransform1dOperator - { - /// - /// Applies the normative eight-point AV1 inverse identity transform. - /// - /// The eight frequency-domain coefficients. - /// The eight scaled spatial-domain values. - /// Unused stage storage supplied by the common transform-kernel contract. - /// Unused cosine precision supplied by the common transform-kernel contract. - /// The signed-bit range assigned to the transform output. - public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) - { - _ = step; - _ = cosBit; - _ = stageRange; - - // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization. - for (int i = 0; i < 8; i++) - { - output[i] = input[i] * 2; - } - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - - /// - public static void Transform( - ref Av1TransformVector> input, - ref Av1TransformVector> output, - ref Av1TransformVector> step, - int cosBit, - Av1TransformStageRange stageRange) - { - Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); - _ = step; - _ = cosBit; - _ = stageRange; - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.OutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.OutputOperator.cs deleted file mode 100644 index c47445aff..000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.OutputOperator.cs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Common.Helpers; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; - -/// -/// Provides the sample-output operator shared by inverse transform traversals. -/// -internal static partial class Av1InverseTransformer -{ - /// - /// Reconstructs AV1 samples from predicted values and inverse-transform residuals. - /// - /// - /// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to - /// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width - /// stores. The closed specialization removes storage-type branches from hot loops. - /// - /// The decoded sample storage type. - internal readonly struct OutputOperator : IAv1InverseTransformOutputOperator - where TSample : unmanaged - { - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static TSample Add(TSample prediction, int residual, int bitDepth) - { - // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each - // closed transform so storage selection does not introduce a branch in the reconstruction loop. - if (typeof(TSample) == typeof(byte)) - { - byte value = (byte)Math.Clamp(Unsafe.As(ref prediction) + residual, byte.MinValue, byte.MaxValue); - return Unsafe.As(ref value); - } - - short result = (short)Math.Clamp(Unsafe.As(ref prediction) + residual, 0, (1 << bitDepth) - 1); - return Unsafe.As(ref result); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth) - { - if (typeof(TSample) == typeof(byte)) - { - // Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach - // memory, which keeps reconstruction valid at a tightly packed row boundary. - ref byte source = ref Unsafe.As(ref prediction); - uint packed = Unsafe.ReadUnaligned(ref source); - Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32(); - Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue)); - Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt32().ToScalar()); - return; - } - - ref short highBitDepthSource = ref Unsafe.As(ref prediction); - ulong highBitDepthPacked = Unsafe.ReadUnaligned(ref Unsafe.As(ref highBitDepthSource)); - Vector128 highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16()); - Vector128 highBitDepthReconstructed = - Vector128.Clamp(highBitDepthPredicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1)); - - Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar()); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth) - { - if (typeof(TSample) == typeof(byte)) - { - // Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store - // covers only those reconstructed samples and does not require destination padding. - ref byte source = ref Unsafe.As(ref prediction); - ulong packed = Unsafe.ReadUnaligned(ref source); - Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32(); - Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue)); - Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); - Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt64().ToScalar()); - return; - } - - ref short highBitDepthSource = ref Unsafe.As(ref prediction); - Vector256 highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource)); - Vector256 highBitDepthReconstructed = - Vector256.Clamp(highBitDepthPredicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1)); - - Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper()); - narrowed.StoreUnsafe(ref Unsafe.As(ref destination)); - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs index 852ac8e73..5ba9ca4c8 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformer.cs @@ -6,7 +6,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// /// Reconstructs decoded AV1 transform coefficients into prediction sample buffers. /// -internal static partial class Av1InverseTransformer +internal class Av1InverseTransformer { /// /// Reconstructs an eight-bit transform block in place by adding its inverse-transform residual. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformerFactory.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformerFactory.cs index c792c96e7..257857443 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformerFactory.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformerFactory.cs @@ -49,7 +49,7 @@ internal static class Av1InverseTransformerFactory transformFunctionParameters.TransformSize, transformFunctionParameters.BitDepth); - Av1InverseTransformer.Transform2dAdd(coefficients, readBuffer, readStride, writeBuffer, writeStride, ref config, workspace); + Av1Inverse2dTransformer.Transform2dAdd(coefficients, readBuffer, readStride, writeBuffer, writeStride, ref config, workspace); } /// @@ -93,6 +93,6 @@ internal static class Av1InverseTransformerFactory transformFunctionParameters.TransformSize, transformFunctionParameters.BitDepth); - Av1InverseTransformer.Transform2dAdd(coefficients, readBuffer, readStride, writeBuffer, writeStride, ref config, workspace, transformFunctionParameters.BitDepth); + Av1Inverse2dTransformer.Transform2dAdd(coefficients, readBuffer, readStride, writeBuffer, writeStride, ref config, workspace, transformFunctionParameters.BitDepth); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs index 921f449e6..034e1d5a4 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs @@ -40,7 +40,7 @@ internal static class Av1InverseWalshHadamardTransformer int writeStride, int coefficientCount, Span workspace) - => TransformAdd>( + => TransformAdd>( coefficients, readBuffer, readStride, @@ -70,7 +70,7 @@ internal static class Av1InverseWalshHadamardTransformer int coefficientCount, Span workspace, int bitDepth) - => TransformAdd>( + => TransformAdd>( coefficients, readBuffer, readStride, @@ -93,7 +93,7 @@ internal static class Av1InverseWalshHadamardTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator + where TOutputOperator : struct, IAv1InverseTransformOutputOperator { if (Vector128.IsHardwareAccelerated) { @@ -116,7 +116,7 @@ internal static class Av1InverseWalshHadamardTransformer int coefficientCount, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator + where TOutputOperator : struct, IAv1InverseTransformOutputOperator { ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); Vector128 row0; @@ -172,7 +172,7 @@ internal static class Av1InverseWalshHadamardTransformer Span workspace, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator + where TOutputOperator : struct, IAv1InverseTransformOutputOperator { ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer); ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer); @@ -300,7 +300,7 @@ internal static class Av1InverseWalshHadamardTransformer Vector128 row3, int bitDepth) where TSample : unmanaged - where TOutputOperator : struct, Av1InverseTransformer.IAv1InverseTransformOutputOperator + where TOutputOperator : struct, IAv1InverseTransformOutputOperator { ref TSample readBase = ref MemoryMarshal.GetReference(readBuffer); ref TSample writeBase = ref MemoryMarshal.GetReference(writeBuffer); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs index 85df71fff..700c8956b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs @@ -17,6 +17,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// internal static class Av1Transform1dMath { + /// + /// The signed stage width whose fixed-point terminal operations require widened SIMD intermediates. + /// + public const byte WidenedIntermediateBitCount = 20; + /// /// The fixed-point representation of the square root of two with twelve fractional bits. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs new file mode 100644 index 000000000..3c354fd6c --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the sixteen-point AV1 forward asymmetric discrete sine transform operator. +/// +internal readonly struct Av1Adst16Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs new file mode 100644 index 000000000..ec561c746 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the four-point AV1 forward asymmetric discrete sine transform operator. +/// +internal readonly struct Av1Adst4Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs new file mode 100644 index 000000000..1d7e0a424 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the eight-point AV1 forward asymmetric discrete sine transform operator. +/// +internal readonly struct Av1Adst8Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs new file mode 100644 index 000000000..e1b447b15 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the sixteen-point AV1 forward discrete cosine transform operator. +/// +internal readonly struct Av1Dct16Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs new file mode 100644 index 000000000..5749ab40e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the thirty-two-point AV1 forward discrete cosine transform operator. +/// +internal readonly struct Av1Dct32Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs new file mode 100644 index 000000000..bef5993f3 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the four-point AV1 forward discrete cosine transform operator. +/// +internal readonly struct Av1Dct4Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs new file mode 100644 index 000000000..ca12aa88a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the sixty-four-point AV1 forward discrete cosine transform operator. +/// +internal readonly struct Av1Dct64Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs new file mode 100644 index 000000000..8fc637085 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the eight-point AV1 forward discrete cosine transform operator. +/// +internal readonly struct Av1Dct8Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs new file mode 100644 index 000000000..5216a8900 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the sixteen-point AV1 forward identity transform operator. +/// +internal readonly struct Av1Identity16Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs new file mode 100644 index 000000000..f15c0b868 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the thirty-two-point AV1 forward identity transform operator. +/// +internal readonly struct Av1Identity32Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs new file mode 100644 index 000000000..89236df3d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the four-point AV1 forward identity transform operator. +/// +internal readonly struct Av1Identity4Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs new file mode 100644 index 000000000..96a295221 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines the eight-point AV1 forward identity transform operator. +/// +internal readonly struct Av1Identity8Forward1dOperator : IAv1ForwardTransform1dOperator +{ + /// + public static void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct + => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs new file mode 100644 index 000000000..6aa68233d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs @@ -0,0 +1,34 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; + +/// +/// Defines one AV1 forward transform which can be specialized for the selected sample and SIMD lane type. +/// +/// +/// A concrete operator identifies the transform stage network. The two-dimensional driver selects the sample type +/// and vector width once per block, allowing the JIT to specialize the complete network without interface dispatch +/// inside the transform stages. +/// +internal interface IAv1ForwardTransform1dOperator +{ + /// + /// Transforms the independent axes stored in each value lane. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + public static abstract void Transform( + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, + int cosBit) + where TValue : struct; +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs new file mode 100644 index 000000000..7e2244cc6 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs @@ -0,0 +1,46 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines how inverse-transform residuals are added to a decoded sample representation. +/// +/// +/// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions, +/// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable +/// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize. +/// +/// The decoded sample storage type. +internal interface IAv1InverseTransformOutputOperator + where TSample : unmanaged +{ + /// + /// Adds one residual to a predicted sample and clips the result to the coded bit depth. + /// + /// The predicted sample. + /// The inverse-transform residual. + /// The coded sample bit depth. + /// The reconstructed sample. + public static abstract TSample Add(TSample prediction, int residual, int bitDepth); + + /// + /// Adds four residuals to four predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The four inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth); + + /// + /// Adds eight residuals to eight predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The eight inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs new file mode 100644 index 000000000..09ecb8a3b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs @@ -0,0 +1,59 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +/// +/// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional transform. +/// +/// +/// Each overload performs the same staged fixed-point transform. In the SIMD overloads, each vector field identifies +/// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore +/// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage +/// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the +/// JIT to specialize every static interface call outside the stage network. +/// +internal interface IAv1Transform1dOperator +{ + /// + /// Transforms one axis when hardware vectorization is unavailable. + /// + /// The source values for the transform axis. + /// The destination values for the transform axis. + /// The fixed stage storage for the transform axis. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange); + + /// + /// Transforms four independent axes in parallel. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The fixed stage storage for four transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); + + /// + /// Transforms eight independent axes in parallel. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The fixed stage storage for eight transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs new file mode 100644 index 000000000..e7e1a2a9f --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs @@ -0,0 +1,568 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Adst16Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative 16-point AV1 inverse asymmetric discrete sine transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen spatial-domain residual values. + /// The sixteen-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[15]; + output[1] = input[0]; + output[2] = input[13]; + output[3] = input[2]; + output[4] = input[11]; + output[5] = input[4]; + output[6] = input[9]; + output[7] = input[6]; + output[8] = input[7]; + output[9] = input[8]; + output[10] = input[5]; + output[11] = input[10]; + output[12] = input[3]; + output[13] = input[12]; + output[14] = input[1]; + output[15] = input[14]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); + step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); + step[12] = output[12]; + step[13] = output[13]; + step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[8]; + output[2] = step[12]; + output[3] = -step[4]; + output[4] = step[6]; + output[5] = -step[14]; + output[6] = step[10]; + output[7] = -step[2]; + output[8] = step[3]; + output[9] = -step[11]; + output[10] = step[15]; + output[11] = -step[7]; + output[12] = step[5]; + output[13] = -step[13]; + output[14] = step[9]; + output[15] = -step[1]; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V15; + output.V1 = input.V0; + output.V2 = input.V13; + output.V3 = input.V2; + output.V4 = input.V11; + output.V5 = input.V4; + output.V6 = input.V9; + output.V7 = input.V6; + output.V8 = input.V7; + output.V9 = input.V8; + output.V10 = input.V5; + output.V11 = input.V10; + output.V12 = input.V3; + output.V13 = input.V12; + output.V14 = input.V1; + output.V15 = input.V14; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V0, cospi[62], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V0, -cospi[2], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V2, cospi[54], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V2, -cospi[10], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V4, cospi[46], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V4, -cospi[18], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V6, cospi[38], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V6, -cospi[26], output.V7, cosBit); + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V8, cospi[30], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V8, -cospi[34], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V10, cospi[22], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V10, -cospi[42], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V12, cospi[14], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V12, -cospi[50], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V14, cospi[6], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V14, -cospi[58], output.V15, cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V8, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V9, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V10, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V11, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V12, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V13, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V14, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V15, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V0 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V1 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V2 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V3 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V4 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V5 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V6 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V7 - step.V15, stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V8, cospi[56], output.V9, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V8, -cospi[8], output.V9, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V10, cospi[24], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V10, -cospi[40], output.V11, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V12, cospi[8], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V12, cospi[56], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V14, cospi[40], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V14, cospi[24], output.V15, cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V12, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V13, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V14, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V15, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V8 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V9 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V10 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V11 - step.V15, stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V12, cospi[48], output.V13, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V12, -cospi[16], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V14, cospi[16], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V14, cospi[48], output.V15, cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V10, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V11, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V8 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V9 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V14, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V15, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V12 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V13 - step.V15, stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V11, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, -cospi[32], output.V11, cosBit); + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, cospi[32], output.V15, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V14, -cospi[32], output.V15, cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V8; + output.V2 = step.V12; + output.V3 = -step.V4; + output.V4 = step.V6; + output.V5 = -step.V14; + output.V6 = step.V10; + output.V7 = -step.V2; + output.V8 = step.V3; + output.V9 = -step.V11; + output.V10 = step.V15; + output.V11 = -step.V7; + output.V12 = step.V5; + output.V13 = -step.V13; + output.V14 = step.V9; + output.V15 = -step.V1; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs new file mode 100644 index 000000000..558064696 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs @@ -0,0 +1,168 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Adst4Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative four-point AV1 inverse asymmetric discrete sine transform. + /// + /// The four frequency-domain coefficients. + /// The four spatial-domain residual values. + /// The stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the sine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + + // libaom widens the complete four-point factorization because the products retain their fixed-point scale + // until the final shift. The stage buffer is therefore unnecessary for this transform size. + long x0 = input[0]; + long x1 = input[1]; + long x2 = input[2]; + long x3 = input[3]; + + _ = step; + _ = stageRange; + + // Avoid the multiplications for the all-zero coefficient vector, matching libaom's scalar kernel. + if ((x0 | x1 | x2 | x3) == 0) + { + output[..4].Clear(); + return; + } + + // Stages 1 and 2 form the seven sine products and the one unscaled combination used by stage 3. + long s0 = sinpi[1] * x0; + long s1 = sinpi[2] * x0; + long s2 = sinpi[3] * x1; + long s3 = sinpi[4] * x2; + long s4 = sinpi[1] * x2; + long s5 = sinpi[2] * x3; + long s6 = sinpi[4] * x3; + long s7 = (x0 - x2) + x3; + + // Stages 3 through 6 combine the products while preserving the fixed-point scale until the final rounding. + s0 += s3; + s1 -= s4; + s3 = s2; + s2 = sinpi[3] * s7; + s0 += s5; + s1 -= s6; + x0 = s0 + s3; + x1 = s1 + s3; + x2 = s2; + x3 = (s0 + s1) - s3; + + output[0] = Av1Math.RoundShift(x0, cosBit); + output[1] = Av1Math.RoundShift(x1, cosBit); + output[2] = Av1Math.RoundShift(x2, cosBit); + output[3] = Av1Math.RoundShift(x3, cosBit); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit, stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount); + _ = step; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit, stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount); + _ = step; + } + + /// + /// Applies the inverse four-point matrix to four independent axes. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The fixed-point precision of the sine constants. + /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. + private static void TransformCore( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + int cosBit, + bool widenedRound) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector128 x0 = input.V0; + Vector128 x1 = input.V1; + Vector128 x2 = input.V2; + Vector128 x3 = input.V3; + + // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row + // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow + // a valid Int32 fixed-point sum. + if (widenedRound) + { + output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + return; + } + + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } + + /// + /// Applies the inverse four-point matrix to eight independent axes. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The fixed-point precision of the sine constants. + /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. + private static void TransformCore( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + int cosBit, + bool widenedRound) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector256 x0 = input.V0; + Vector256 x1 = input.V1; + Vector256 x2 = input.V2; + Vector256 x3 = input.V3; + + if (widenedRound) + { + output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + return; + } + + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs new file mode 100644 index 000000000..0f998e26d --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs @@ -0,0 +1,289 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Adst8Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative eight-point AV1 inverse asymmetric discrete sine transform. + /// + /// The eight frequency-domain coefficients. + /// The eight spatial-domain residual values. + /// The eight-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[7]; + output[1] = input[0]; + output[2] = input[5]; + output[3] = input[2]; + output[4] = input[3]; + output[5] = input[4]; + output[6] = input[1]; + output[7] = input[6]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[4]; + output[2] = step[6]; + output[3] = -step[2]; + output[4] = step[3]; + output[5] = -step[7]; + output[6] = step[5]; + output[7] = -step[1]; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output.V0 = input.V7; + output.V1 = input.V0; + output.V2 = input.V5; + output.V3 = input.V2; + output.V4 = input.V3; + output.V5 = input.V4; + output.V6 = input.V1; + output.V7 = input.V6; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V0, cospi[60], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V0, -cospi[4], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V2, cospi[44], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V2, -cospi[20], output.V3, cosBit); + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V4, cospi[28], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V4, -cospi[36], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V6, cospi[12], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V6, -cospi[52], output.V7, cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V4, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V5, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V6, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V7, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V0 - step.V4, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V1 - step.V5, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V2 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V3 - step.V7, stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V4, cospi[48], output.V5, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V4, -cospi[16], output.V5, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V6, cospi[16], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V6, cospi[48], output.V7, cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V2, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V3, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V0 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V1 - step.V3, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V6, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V7, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V4 - step.V6, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V5 - step.V7, stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, cospi[32], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V2, -cospi[32], output.V3, cosBit); + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, cospi[32], output.V7, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V6, -cospi[32], output.V7, cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output.V0 = step.V0; + output.V1 = -step.V4; + output.V2 = step.V6; + output.V3 = -step.V2; + output.V4 = step.V3; + output.V5 = -step.V7; + output.V6 = step.V5; + output.V7 = -step.V1; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs new file mode 100644 index 000000000..442177986 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs @@ -0,0 +1,475 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the 16-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Dct16Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative 16-point AV1 inverse discrete cosine transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen spatial-domain residual values. + /// The sixteen-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[8]; + output[2] = input[4]; + output[3] = input[12]; + output[4] = input[2]; + output[5] = input[10]; + output[6] = input[6]; + output[7] = input[14]; + output[8] = input[1]; + output[9] = input[9]; + output[10] = input[5]; + output[11] = input[13]; + output[12] = input[3]; + output[13] = input[11]; + output[14] = input[7]; + output[15] = input[15]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V8; + output.V2 = input.V4; + output.V3 = input.V12; + output.V4 = input.V2; + output.V5 = input.V10; + output.V6 = input.V6; + output.V7 = input.V14; + output.V8 = input.V1; + output.V9 = input.V9; + output.V10 = input.V5; + output.V11 = input.V13; + output.V12 = input.V3; + output.V13 = input.V11; + output.V14 = input.V7; + output.V15 = input.V15; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, range); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V11 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, range); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V15 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, range); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V7 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, range); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, range); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V15 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V14 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, range); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, range); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, range); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, range); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, range); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, range); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, range); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, range); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, range); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, range); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, range); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, range); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, range); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, range); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs new file mode 100644 index 000000000..d768f9620 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs @@ -0,0 +1,1027 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the 32-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Dct32Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative 32-point AV1 inverse discrete cosine transform. + /// + /// The 32 frequency-domain coefficients. + /// The 32 spatial-domain residual values. + /// The 32-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[16]; + output[2] = input[8]; + output[3] = input[24]; + output[4] = input[4]; + output[5] = input[20]; + output[6] = input[12]; + output[7] = input[28]; + output[8] = input[2]; + output[9] = input[18]; + output[10] = input[10]; + output[11] = input[26]; + output[12] = input[6]; + output[13] = input[22]; + output[14] = input[14]; + output[15] = input[30]; + output[16] = input[1]; + output[17] = input[17]; + output[18] = input[9]; + output[19] = input[25]; + output[20] = input[5]; + output[21] = input[21]; + output[22] = input[13]; + output[23] = input[29]; + output[24] = input[3]; + output[25] = input[19]; + output[26] = input[11]; + output[27] = input[27]; + output[28] = input[7]; + output[29] = input[23]; + output[30] = input[15]; + output[31] = input[31]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); + step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); + step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); + output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); + output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); + output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); + output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); + output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); + output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); + output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); + output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); + output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); + output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); + step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); + step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); + step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); + step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); + step[16] = output[16]; + step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); + step[19] = output[19]; + step[20] = output[20]; + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); + step[23] = output[23]; + step[24] = output[24]; + step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); + step[27] = output[27]; + step[28] = output[28]; + step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); + step[31] = output[31]; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + output[8] = step[8]; + output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); + output[11] = step[11]; + output[12] = step[12]; + output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); + output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); + output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); + output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); + output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); + output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); + output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); + step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); + step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); + step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); + step[30] = output[30]; + step[31] = output[31]; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + output[8] = step[8]; + output[9] = step[9]; + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); + output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); + output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); + output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); + output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); + step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); + step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); + step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); + output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); + output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); + output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V16; + output.V2 = input.V8; + output.V3 = input.V24; + output.V4 = input.V4; + output.V5 = input.V20; + output.V6 = input.V12; + output.V7 = input.V28; + output.V8 = input.V2; + output.V9 = input.V18; + output.V10 = input.V10; + output.V11 = input.V26; + output.V12 = input.V6; + output.V13 = input.V22; + output.V14 = input.V14; + output.V15 = input.V30; + output.V16 = input.V1; + output.V17 = input.V17; + output.V18 = input.V9; + output.V19 = input.V25; + output.V20 = input.V5; + output.V21 = input.V21; + output.V22 = input.V13; + output.V23 = input.V29; + output.V24 = input.V3; + output.V25 = input.V19; + output.V26 = input.V11; + output.V27 = input.V27; + output.V28 = input.V7; + output.V29 = input.V23; + output.V30 = input.V15; + output.V31 = input.V31; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], output.V16, -cospi[2], output.V31, cosBit); + step.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], output.V17, -cospi[34], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], output.V18, -cospi[18], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], output.V19, -cospi[50], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], output.V20, -cospi[10], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], output.V21, -cospi[42], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], output.V22, -cospi[26], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], output.V23, -cospi[58], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], output.V23, cospi[6], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], output.V22, cospi[38], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], output.V21, cospi[22], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], output.V20, cospi[54], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], output.V19, cospi[14], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], output.V18, cospi[46], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], output.V17, cospi[30], output.V30, cosBit); + step.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], output.V16, cospi[62], output.V31, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], step.V8, -cospi[4], step.V15, cosBit); + output.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], step.V9, -cospi[36], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], step.V10, -cospi[20], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], step.V11, -cospi[52], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], step.V11, cospi[12], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], step.V10, cospi[44], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], step.V9, cospi[28], step.V14, cosBit); + output.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], step.V8, cospi[60], step.V15, cosBit); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V17, range); + output.V17 = Av1Transform1dMath.Clamp(step.V16 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(-step.V18 + step.V19, range); + output.V19 = Av1Transform1dMath.Clamp(step.V18 + step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V21, range); + output.V21 = Av1Transform1dMath.Clamp(step.V20 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(-step.V22 + step.V23, range); + output.V23 = Av1Transform1dMath.Clamp(step.V22 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V25, range); + output.V25 = Av1Transform1dMath.Clamp(step.V24 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V27, range); + output.V27 = Av1Transform1dMath.Clamp(step.V26 + step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V29, range); + output.V29 = Av1Transform1dMath.Clamp(step.V28 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(-step.V30 + step.V31, range); + output.V31 = Av1Transform1dMath.Clamp(step.V30 + step.V31, range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], step.V7, cosBit); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V9, range); + step.V9 = Av1Transform1dMath.Clamp(output.V8 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(-output.V10 + output.V11, range); + step.V11 = Av1Transform1dMath.Clamp(output.V10 + output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V13, range); + step.V13 = Av1Transform1dMath.Clamp(output.V12 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(-output.V14 + output.V15, range); + step.V15 = Av1Transform1dMath.Clamp(output.V14 + output.V15, range); + step.V16 = output.V16; + step.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V17, cospi[56], output.V30, cosBit); + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V18, -cospi[8], output.V29, cosBit); + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V21, cospi[24], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V22, -cospi[40], output.V25, cosBit); + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V22, cospi[24], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V21, cospi[40], output.V26, cosBit); + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V18, cospi[56], output.V29, cosBit); + step.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V17, cospi[8], output.V30, cosBit); + step.V31 = output.V31; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(-step.V6 + step.V7, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + output.V8 = step.V8; + output.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V9, cospi[48], step.V14, cosBit); + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V10, -cospi[16], step.V13, cosBit); + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V10, cospi[48], step.V13, cosBit); + output.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V9, cospi[16], step.V14, cosBit); + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V19, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V18, range); + output.V18 = Av1Transform1dMath.Clamp(step.V17 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V16 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(-step.V20 + step.V23, range); + output.V21 = Av1Transform1dMath.Clamp(-step.V21 + step.V22, range); + output.V22 = Av1Transform1dMath.Clamp(step.V21 + step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V20 + step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V27, range); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V26, range); + output.V26 = Av1Transform1dMath.Clamp(step.V25 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V24 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(-step.V28 + step.V31, range); + output.V29 = Av1Transform1dMath.Clamp(-step.V29 + step.V30, range); + output.V30 = Av1Transform1dMath.Clamp(step.V29 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V28 + step.V31, range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V11, range); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V10, range); + step.V10 = Av1Transform1dMath.Clamp(output.V9 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V8 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(-output.V12 + output.V15, range); + step.V13 = Av1Transform1dMath.Clamp(-output.V13 + output.V14, range); + step.V14 = Av1Transform1dMath.Clamp(output.V13 + output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V12 + output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V18, cospi[48], output.V29, cosBit); + step.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V19, cospi[48], output.V28, cosBit); + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V20, -cospi[16], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V21, -cospi[16], output.V26, cosBit); + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V21, cospi[48], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V20, cospi[48], output.V27, cosBit); + step.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V19, cospi[16], output.V28, cosBit); + step.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V18, cospi[16], output.V29, cosBit); + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V11, cospi[32], step.V12, cosBit); + output.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V10, cospi[32], step.V13, cosBit); + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V23, range); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V22, range); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V21, range); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V20, range); + output.V20 = Av1Transform1dMath.Clamp(step.V19 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V18 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V17 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V16 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(-step.V24 + step.V31, range); + output.V25 = Av1Transform1dMath.Clamp(-step.V25 + step.V30, range); + output.V26 = Av1Transform1dMath.Clamp(-step.V26 + step.V29, range); + output.V27 = Av1Transform1dMath.Clamp(-step.V27 + step.V28, range); + output.V28 = Av1Transform1dMath.Clamp(step.V27 + step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V26 + step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V25 + step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V24 + step.V31, range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V15, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V14, range); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V13, range); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V12, range); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V11, range); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V10, range); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V9, range); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V8, range); + step.V8 = Av1Transform1dMath.Clamp(output.V7 - output.V8, range); + step.V9 = Av1Transform1dMath.Clamp(output.V6 - output.V9, range); + step.V10 = Av1Transform1dMath.Clamp(output.V5 - output.V10, range); + step.V11 = Av1Transform1dMath.Clamp(output.V4 - output.V11, range); + step.V12 = Av1Transform1dMath.Clamp(output.V3 - output.V12, range); + step.V13 = Av1Transform1dMath.Clamp(output.V2 - output.V13, range); + step.V14 = Av1Transform1dMath.Clamp(output.V1 - output.V14, range); + step.V15 = Av1Transform1dMath.Clamp(output.V0 - output.V15, range); + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V23, cospi[32], output.V24, cosBit); + step.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V22, cospi[32], output.V25, cosBit); + step.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V21, cospi[32], output.V26, cosBit); + step.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V20, cospi[32], output.V27, cosBit); + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V31, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V30, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V29, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V28, range); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V27, range); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V26, range); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V25, range); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V24, range); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V23, range); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V22, range); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V21, range); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V20, range); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V19, range); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V18, range); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V17, range); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V16, range); + output.V16 = Av1Transform1dMath.Clamp(step.V15 - step.V16, range); + output.V17 = Av1Transform1dMath.Clamp(step.V14 - step.V17, range); + output.V18 = Av1Transform1dMath.Clamp(step.V13 - step.V18, range); + output.V19 = Av1Transform1dMath.Clamp(step.V12 - step.V19, range); + output.V20 = Av1Transform1dMath.Clamp(step.V11 - step.V20, range); + output.V21 = Av1Transform1dMath.Clamp(step.V10 - step.V21, range); + output.V22 = Av1Transform1dMath.Clamp(step.V9 - step.V22, range); + output.V23 = Av1Transform1dMath.Clamp(step.V8 - step.V23, range); + output.V24 = Av1Transform1dMath.Clamp(step.V7 - step.V24, range); + output.V25 = Av1Transform1dMath.Clamp(step.V6 - step.V25, range); + output.V26 = Av1Transform1dMath.Clamp(step.V5 - step.V26, range); + output.V27 = Av1Transform1dMath.Clamp(step.V4 - step.V27, range); + output.V28 = Av1Transform1dMath.Clamp(step.V3 - step.V28, range); + output.V29 = Av1Transform1dMath.Clamp(step.V2 - step.V29, range); + output.V30 = Av1Transform1dMath.Clamp(step.V1 - step.V30, range); + output.V31 = Av1Transform1dMath.Clamp(step.V0 - step.V31, range); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs new file mode 100644 index 000000000..e638f6a06 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs @@ -0,0 +1,112 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the four-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Dct4Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative four-point AV1 inverse discrete cosine transform. + /// + /// The four frequency-domain coefficients. + /// The four spatial-domain residual values. + /// The four-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output[0] = input[0]; + output[1] = input[2]; + output[2] = input[1]; + output[3] = input[3]; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output.V0 = input.V0; + output.V1 = input.V2; + output.V2 = input.V1; + output.V3 = input.V3; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, range); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, range); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, range); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs new file mode 100644 index 000000000..c532aaed1 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs @@ -0,0 +1,2272 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the 64-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Dct64Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative 64-point AV1 inverse discrete cosine transform. + /// + /// The 64 frequency-domain coefficients. + /// The 64 spatial-domain residual values. + /// The 64-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[32]; + output[2] = input[16]; + output[3] = input[48]; + output[4] = input[8]; + output[5] = input[40]; + output[6] = input[24]; + output[7] = input[56]; + output[8] = input[4]; + output[9] = input[36]; + output[10] = input[20]; + output[11] = input[52]; + output[12] = input[12]; + output[13] = input[44]; + output[14] = input[28]; + output[15] = input[60]; + output[16] = input[2]; + output[17] = input[34]; + output[18] = input[18]; + output[19] = input[50]; + output[20] = input[10]; + output[21] = input[42]; + output[22] = input[26]; + output[23] = input[58]; + output[24] = input[6]; + output[25] = input[38]; + output[26] = input[22]; + output[27] = input[54]; + output[28] = input[14]; + output[29] = input[46]; + output[30] = input[30]; + output[31] = input[62]; + output[32] = input[1]; + output[33] = input[33]; + output[34] = input[17]; + output[35] = input[49]; + output[36] = input[9]; + output[37] = input[41]; + output[38] = input[25]; + output[39] = input[57]; + output[40] = input[5]; + output[41] = input[37]; + output[42] = input[21]; + output[43] = input[53]; + output[44] = input[13]; + output[45] = input[45]; + output[46] = input[29]; + output[47] = input[61]; + output[48] = input[3]; + output[49] = input[35]; + output[50] = input[19]; + output[51] = input[51]; + output[52] = input[11]; + output[53] = input[43]; + output[54] = input[27]; + output[55] = input[59]; + output[56] = input[7]; + output[57] = input[39]; + output[58] = input[23]; + output[59] = input[55]; + output[60] = input[15]; + output[61] = input[47]; + output[62] = input[31]; + output[63] = input[63]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = output[20]; + step[21] = output[21]; + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = output[26]; + step[27] = output[27]; + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); + step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); + step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = step[10]; + output[11] = step[11]; + output[12] = step[12]; + output[13] = step[13]; + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); + output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); + output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); + output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); + step[35] = output[35]; + step[36] = output[36]; + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); + step[39] = output[39]; + step[40] = output[40]; + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); + step[43] = output[43]; + step[44] = output[44]; + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); + step[47] = output[47]; + step[48] = output[48]; + step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); + step[51] = output[51]; + step[52] = output[52]; + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); + step[55] = output[55]; + step[56] = output[56]; + step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); + step[59] = output[59]; + step[60] = output[60]; + step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); + step[63] = output[63]; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); + output[19] = step[19]; + output[20] = step[20]; + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); + output[23] = step[23]; + output[24] = step[24]; + output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); + output[27] = step[27]; + output[28] = step[28]; + output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); + step[38] = output[38]; + step[39] = output[39]; + step[40] = output[40]; + step[41] = output[41]; + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); + step[54] = output[54]; + step[55] = output[55]; + step[56] = output[56]; + step[57] = output[57]; + step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); + step[62] = output[62]; + step[63] = output[63]; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); + step[44] = output[44]; + step[45] = output[45]; + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = output[50]; + step[51] = output[51]; + step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); + step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); + step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); + step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); + step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); + step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); + step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = output[36]; + step[37] = output[37]; + step[38] = output[38]; + step[39] = output[39]; + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); + step[56] = output[56]; + step[57] = output[57]; + step[58] = output[58]; + step[59] = output[59]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); + output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); + output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); + output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); + output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); + output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); + output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); + output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); + output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); + output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V32; + output.V2 = input.V16; + output.V3 = input.V48; + output.V4 = input.V8; + output.V5 = input.V40; + output.V6 = input.V24; + output.V7 = input.V56; + output.V8 = input.V4; + output.V9 = input.V36; + output.V10 = input.V20; + output.V11 = input.V52; + output.V12 = input.V12; + output.V13 = input.V44; + output.V14 = input.V28; + output.V15 = input.V60; + output.V16 = input.V2; + output.V17 = input.V34; + output.V18 = input.V18; + output.V19 = input.V50; + output.V20 = input.V10; + output.V21 = input.V42; + output.V22 = input.V26; + output.V23 = input.V58; + output.V24 = input.V6; + output.V25 = input.V38; + output.V26 = input.V22; + output.V27 = input.V54; + output.V28 = input.V14; + output.V29 = input.V46; + output.V30 = input.V30; + output.V31 = input.V62; + output.V32 = input.V1; + output.V33 = input.V33; + output.V34 = input.V17; + output.V35 = input.V49; + output.V36 = input.V9; + output.V37 = input.V41; + output.V38 = input.V25; + output.V39 = input.V57; + output.V40 = input.V5; + output.V41 = input.V37; + output.V42 = input.V21; + output.V43 = input.V53; + output.V44 = input.V13; + output.V45 = input.V45; + output.V46 = input.V29; + output.V47 = input.V61; + output.V48 = input.V3; + output.V49 = input.V35; + output.V50 = input.V19; + output.V51 = input.V51; + output.V52 = input.V11; + output.V53 = input.V43; + output.V54 = input.V27; + output.V55 = input.V59; + output.V56 = input.V7; + output.V57 = input.V39; + output.V58 = input.V23; + output.V59 = input.V55; + output.V60 = input.V15; + output.V61 = input.V47; + output.V62 = input.V31; + output.V63 = input.V63; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = output.V10; + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = output.V13; + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = output.V16; + step.V17 = output.V17; + step.V18 = output.V18; + step.V19 = output.V19; + step.V20 = output.V20; + step.V21 = output.V21; + step.V22 = output.V22; + step.V23 = output.V23; + step.V24 = output.V24; + step.V25 = output.V25; + step.V26 = output.V26; + step.V27 = output.V27; + step.V28 = output.V28; + step.V29 = output.V29; + step.V30 = output.V30; + step.V31 = output.V31; + step.V32 = Av1Transform1dMath.HalfButterfly(cospi[63], output.V32, -cospi[1], output.V63, cosBit); + step.V33 = Av1Transform1dMath.HalfButterfly(cospi[31], output.V33, -cospi[33], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(cospi[47], output.V34, -cospi[17], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(cospi[15], output.V35, -cospi[49], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(cospi[55], output.V36, -cospi[9], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(cospi[23], output.V37, -cospi[41], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(cospi[39], output.V38, -cospi[25], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(cospi[7], output.V39, -cospi[57], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(cospi[59], output.V40, -cospi[5], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(cospi[27], output.V41, -cospi[37], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(cospi[43], output.V42, -cospi[21], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(cospi[11], output.V43, -cospi[53], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(cospi[51], output.V44, -cospi[13], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(cospi[19], output.V45, -cospi[45], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(cospi[35], output.V46, -cospi[29], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(cospi[3], output.V47, -cospi[61], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[61], output.V47, cospi[3], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[29], output.V46, cospi[35], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[45], output.V45, cospi[19], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[13], output.V44, cospi[51], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[53], output.V43, cospi[11], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[21], output.V42, cospi[43], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[37], output.V41, cospi[27], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[5], output.V40, cospi[59], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[57], output.V39, cospi[7], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[25], output.V38, cospi[39], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[41], output.V37, cospi[23], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[9], output.V36, cospi[55], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[49], output.V35, cospi[15], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[17], output.V34, cospi[47], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[33], output.V33, cospi[31], output.V62, cosBit); + step.V63 = Av1Transform1dMath.HalfButterfly(cospi[1], output.V32, cospi[63], output.V63, cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = step.V4; + output.V5 = step.V5; + output.V6 = step.V6; + output.V7 = step.V7; + output.V8 = step.V8; + output.V9 = step.V9; + output.V10 = step.V10; + output.V11 = step.V11; + output.V12 = step.V12; + output.V13 = step.V13; + output.V14 = step.V14; + output.V15 = step.V15; + output.V16 = Av1Transform1dMath.HalfButterfly(cospi[62], step.V16, -cospi[2], step.V31, cosBit); + output.V17 = Av1Transform1dMath.HalfButterfly(cospi[30], step.V17, -cospi[34], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(cospi[46], step.V18, -cospi[18], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(cospi[14], step.V19, -cospi[50], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(cospi[54], step.V20, -cospi[10], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(cospi[22], step.V21, -cospi[42], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(cospi[38], step.V22, -cospi[26], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(cospi[6], step.V23, -cospi[58], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[58], step.V23, cospi[6], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[26], step.V22, cospi[38], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[42], step.V21, cospi[22], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[10], step.V20, cospi[54], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[50], step.V19, cospi[14], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[18], step.V18, cospi[46], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[34], step.V17, cospi[30], step.V30, cosBit); + output.V31 = Av1Transform1dMath.HalfButterfly(cospi[2], step.V16, cospi[62], step.V31, cosBit); + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V33, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V32 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(-step.V34 + step.V35, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V34 + step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V37, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V36 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(-step.V38 + step.V39, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V38 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V41, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V40 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V43, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V42 + step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V44 + step.V45, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V44 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(-step.V46 + step.V47, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V46 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V49, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V48 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V51, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V50 + step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V52 + step.V53, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V52 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V55, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V54 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V57, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V56 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V59, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V58 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V60 + step.V61, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V60 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(-step.V62 + step.V63, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V62 + step.V63, stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = output.V4; + step.V5 = output.V5; + step.V6 = output.V6; + step.V7 = output.V7; + step.V8 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V8, -cospi[4], output.V15, cosBit); + step.V9 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V9, -cospi[36], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V10, -cospi[20], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V11, -cospi[52], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[52], output.V11, cospi[12], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[20], output.V10, cospi[44], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[36], output.V9, cospi[28], output.V14, cosBit); + step.V15 = Av1Transform1dMath.HalfButterfly(cospi[4], output.V8, cospi[60], output.V15, cosBit); + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V17, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V16 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(-output.V18 + output.V19, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V18 + output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V20 + output.V21, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V20 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(-output.V22 + output.V23, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V22 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V25, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V24 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V27, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V26 + output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V28 + output.V29, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V28 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(-output.V30 + output.V31, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V30 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V33, cospi[60], output.V62, cosBit); + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[60], output.V34, -cospi[4], output.V61, cosBit); + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V37, cospi[28], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[28], output.V38, -cospi[36], output.V57, cosBit); + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V41, cospi[44], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[44], output.V42, -cospi[20], output.V53, cosBit); + step.V43 = output.V43; + step.V44 = output.V44; + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V45, cospi[12], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[12], output.V46, -cospi[52], output.V49, cosBit); + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = Av1Transform1dMath.HalfButterfly(-cospi[52], output.V46, cospi[12], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[12], output.V45, cospi[52], output.V50, cosBit); + step.V51 = output.V51; + step.V52 = output.V52; + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[20], output.V42, cospi[44], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[44], output.V41, cospi[20], output.V54, cosBit); + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = Av1Transform1dMath.HalfButterfly(-cospi[36], output.V38, cospi[28], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[28], output.V37, cospi[36], output.V58, cosBit); + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = Av1Transform1dMath.HalfButterfly(-cospi[4], output.V34, cospi[60], output.V61, cosBit); + step.V62 = Av1Transform1dMath.HalfButterfly(cospi[60], output.V33, cospi[4], output.V62, cosBit); + step.V63 = output.V63; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output.V0 = step.V0; + output.V1 = step.V1; + output.V2 = step.V2; + output.V3 = step.V3; + output.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V4, -cospi[8], step.V7, cosBit); + output.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V5, -cospi[40], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], step.V5, cospi[24], step.V6, cosBit); + output.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], step.V4, cospi[56], step.V7, cosBit); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V9, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V8 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(-step.V10 + step.V11, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V10 + step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V13, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V12 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(-step.V14 + step.V15, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V14 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V17, cospi[56], step.V30, cosBit); + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[56], step.V18, -cospi[8], step.V29, cosBit); + output.V19 = step.V19; + output.V20 = step.V20; + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V21, cospi[24], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[24], step.V22, -cospi[40], step.V25, cosBit); + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = Av1Transform1dMath.HalfButterfly(-cospi[40], step.V22, cospi[24], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[24], step.V21, cospi[40], step.V26, cosBit); + output.V27 = step.V27; + output.V28 = step.V28; + output.V29 = Av1Transform1dMath.HalfButterfly(-cospi[8], step.V18, cospi[56], step.V29, cosBit); + output.V30 = Av1Transform1dMath.HalfButterfly(cospi[56], step.V17, cospi[8], step.V30, cosBit); + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V35, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V34, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V33 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V32 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(-step.V36 + step.V39, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(-step.V37 + step.V38, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V37 + step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V36 + step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V40 + step.V43, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V41 + step.V42, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V41 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V40 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(-step.V44 + step.V47, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(-step.V45 + step.V46, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V45 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V44 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V51, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V50, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V49 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V48 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V55, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V54, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V53 + step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V52 + step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V56 + step.V59, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V57 + step.V58, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V57 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V56 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(-step.V60 + step.V63, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(-step.V61 + step.V62, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V61 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V60 + step.V63, stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, cospi[32], output.V1, cosBit); + step.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V0, -cospi[32], output.V1, cosBit); + step.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V2, -cospi[16], output.V3, cosBit); + step.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], output.V2, cospi[48], output.V3, cosBit); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V5, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V4 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(-output.V6 + output.V7, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V6 + output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V9, cospi[48], output.V14, cosBit); + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V10, -cospi[16], output.V13, cosBit); + step.V11 = output.V11; + step.V12 = output.V12; + step.V13 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V10, cospi[48], output.V13, cosBit); + step.V14 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V9, cospi[16], output.V14, cosBit); + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V19, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V18, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V17 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V16 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(-output.V20 + output.V23, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(-output.V21 + output.V22, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V21 + output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V20 + output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V24 + output.V27, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V25 + output.V26, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V25 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V24 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(-output.V28 + output.V31, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(-output.V29 + output.V30, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V29 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V28 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V34, cospi[56], output.V61, cosBit); + step.V35 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V35, cospi[56], output.V60, cosBit); + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V36, -cospi[8], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[56], output.V37, -cospi[8], output.V58, cosBit); + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = output.V40; + step.V41 = output.V41; + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V42, cospi[24], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V43, cospi[24], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V44, -cospi[40], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[24], output.V45, -cospi[40], output.V50, cosBit); + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V45, cospi[24], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(-cospi[40], output.V44, cospi[24], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V43, cospi[40], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V42, cospi[40], output.V53, cosBit); + step.V54 = output.V54; + step.V55 = output.V55; + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V37, cospi[56], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(-cospi[8], output.V36, cospi[56], output.V59, cosBit); + step.V60 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V35, cospi[8], output.V60, cosBit); + step.V61 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V34, cospi[8], output.V61, cosBit); + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V3, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V2, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V1 - step.V2, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V0 - step.V3, stageRange[stage]); + output.V4 = step.V4; + output.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V5, cospi[32], step.V6, cosBit); + output.V7 = step.V7; + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V11, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V10, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V9 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V8 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(-step.V12 + step.V15, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(-step.V13 + step.V14, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V13 + step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V12 + step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V18, cospi[48], step.V29, cosBit); + output.V19 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V19, cospi[48], step.V28, cosBit); + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V20, -cospi[16], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[48], step.V21, -cospi[16], step.V26, cosBit); + output.V22 = step.V22; + output.V23 = step.V23; + output.V24 = step.V24; + output.V25 = step.V25; + output.V26 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V21, cospi[48], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(-cospi[16], step.V20, cospi[48], step.V27, cosBit); + output.V28 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V19, cospi[16], step.V28, cosBit); + output.V29 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V18, cospi[16], step.V29, cosBit); + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V39, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V38, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V37, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V36, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V35 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V34 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V33 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V32 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(-step.V40 + step.V47, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(-step.V41 + step.V46, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(-step.V42 + step.V45, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(-step.V43 + step.V44, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V43 + step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V42 + step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V41 + step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V40 + step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V48 + step.V55, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V49 + step.V54, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V50 + step.V53, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V51 + step.V52, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V51 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V50 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V49 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V48 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(-step.V56 + step.V63, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(-step.V57 + step.V62, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(-step.V58 + step.V61, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(-step.V59 + step.V60, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V59 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V58 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V57 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V56 + step.V63, stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V7, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V6, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V5, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V4, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V3 - output.V4, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V2 - output.V5, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V1 - output.V6, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V0 - output.V7, stageRange[stage]); + step.V8 = output.V8; + step.V9 = output.V9; + step.V10 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V11 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V12 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V11, cospi[32], output.V12, cosBit); + step.V13 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V10, cospi[32], output.V13, cosBit); + step.V14 = output.V14; + step.V15 = output.V15; + step.V16 = Av1Transform1dMath.Clamp(output.V16 + output.V23, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V17 + output.V22, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V18 + output.V21, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V19 + output.V20, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V19 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V18 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V17 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V16 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(-output.V24 + output.V31, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(-output.V25 + output.V30, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(-output.V26 + output.V29, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(-output.V27 + output.V28, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V27 + output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V26 + output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V25 + output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V24 + output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V36, cospi[48], output.V59, cosBit); + step.V37 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V37, cospi[48], output.V58, cosBit); + step.V38 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V38, cospi[48], output.V57, cosBit); + step.V39 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V39, cospi[48], output.V56, cosBit); + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V40, -cospi[16], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V41, -cospi[16], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V42, -cospi[16], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[48], output.V43, -cospi[16], output.V52, cosBit); + step.V44 = output.V44; + step.V45 = output.V45; + step.V46 = output.V46; + step.V47 = output.V47; + step.V48 = output.V48; + step.V49 = output.V49; + step.V50 = output.V50; + step.V51 = output.V51; + step.V52 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V43, cospi[48], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V42, cospi[48], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V41, cospi[48], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(-cospi[16], output.V40, cospi[48], output.V55, cosBit); + step.V56 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V39, cospi[16], output.V56, cosBit); + step.V57 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V38, cospi[16], output.V57, cosBit); + step.V58 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V37, cospi[16], output.V58, cosBit); + step.V59 = Av1Transform1dMath.HalfButterfly(cospi[48], output.V36, cospi[16], output.V59, cosBit); + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V15, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V14, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V13, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V12, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V11, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V10, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V9, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V8, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V7 - step.V8, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V6 - step.V9, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V5 - step.V10, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V4 - step.V11, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V3 - step.V12, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V2 - step.V13, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V1 - step.V14, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V0 - step.V15, stageRange[stage]); + output.V16 = step.V16; + output.V17 = step.V17; + output.V18 = step.V18; + output.V19 = step.V19; + output.V20 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V21 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V22 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V23 = Av1Transform1dMath.HalfButterfly(-cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V24 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V23, cospi[32], step.V24, cosBit); + output.V25 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V22, cospi[32], step.V25, cosBit); + output.V26 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V21, cospi[32], step.V26, cosBit); + output.V27 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V20, cospi[32], step.V27, cosBit); + output.V28 = step.V28; + output.V29 = step.V29; + output.V30 = step.V30; + output.V31 = step.V31; + output.V32 = Av1Transform1dMath.Clamp(step.V32 + step.V47, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V33 + step.V46, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V34 + step.V45, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V35 + step.V44, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V36 + step.V43, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V37 + step.V42, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V38 + step.V41, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V39 + step.V40, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V39 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V38 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V37 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V36 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V35 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V34 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V33 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V32 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(-step.V48 + step.V63, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(-step.V49 + step.V62, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(-step.V50 + step.V61, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(-step.V51 + step.V60, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(-step.V52 + step.V59, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(-step.V53 + step.V58, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(-step.V54 + step.V57, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(-step.V55 + step.V56, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V55 + step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V54 + step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V53 + step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V52 + step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V51 + step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V50 + step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V49 + step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V48 + step.V63, stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V31, stageRange[stage]); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V30, stageRange[stage]); + step.V2 = Av1Transform1dMath.Clamp(output.V2 + output.V29, stageRange[stage]); + step.V3 = Av1Transform1dMath.Clamp(output.V3 + output.V28, stageRange[stage]); + step.V4 = Av1Transform1dMath.Clamp(output.V4 + output.V27, stageRange[stage]); + step.V5 = Av1Transform1dMath.Clamp(output.V5 + output.V26, stageRange[stage]); + step.V6 = Av1Transform1dMath.Clamp(output.V6 + output.V25, stageRange[stage]); + step.V7 = Av1Transform1dMath.Clamp(output.V7 + output.V24, stageRange[stage]); + step.V8 = Av1Transform1dMath.Clamp(output.V8 + output.V23, stageRange[stage]); + step.V9 = Av1Transform1dMath.Clamp(output.V9 + output.V22, stageRange[stage]); + step.V10 = Av1Transform1dMath.Clamp(output.V10 + output.V21, stageRange[stage]); + step.V11 = Av1Transform1dMath.Clamp(output.V11 + output.V20, stageRange[stage]); + step.V12 = Av1Transform1dMath.Clamp(output.V12 + output.V19, stageRange[stage]); + step.V13 = Av1Transform1dMath.Clamp(output.V13 + output.V18, stageRange[stage]); + step.V14 = Av1Transform1dMath.Clamp(output.V14 + output.V17, stageRange[stage]); + step.V15 = Av1Transform1dMath.Clamp(output.V15 + output.V16, stageRange[stage]); + step.V16 = Av1Transform1dMath.Clamp(output.V15 - output.V16, stageRange[stage]); + step.V17 = Av1Transform1dMath.Clamp(output.V14 - output.V17, stageRange[stage]); + step.V18 = Av1Transform1dMath.Clamp(output.V13 - output.V18, stageRange[stage]); + step.V19 = Av1Transform1dMath.Clamp(output.V12 - output.V19, stageRange[stage]); + step.V20 = Av1Transform1dMath.Clamp(output.V11 - output.V20, stageRange[stage]); + step.V21 = Av1Transform1dMath.Clamp(output.V10 - output.V21, stageRange[stage]); + step.V22 = Av1Transform1dMath.Clamp(output.V9 - output.V22, stageRange[stage]); + step.V23 = Av1Transform1dMath.Clamp(output.V8 - output.V23, stageRange[stage]); + step.V24 = Av1Transform1dMath.Clamp(output.V7 - output.V24, stageRange[stage]); + step.V25 = Av1Transform1dMath.Clamp(output.V6 - output.V25, stageRange[stage]); + step.V26 = Av1Transform1dMath.Clamp(output.V5 - output.V26, stageRange[stage]); + step.V27 = Av1Transform1dMath.Clamp(output.V4 - output.V27, stageRange[stage]); + step.V28 = Av1Transform1dMath.Clamp(output.V3 - output.V28, stageRange[stage]); + step.V29 = Av1Transform1dMath.Clamp(output.V2 - output.V29, stageRange[stage]); + step.V30 = Av1Transform1dMath.Clamp(output.V1 - output.V30, stageRange[stage]); + step.V31 = Av1Transform1dMath.Clamp(output.V0 - output.V31, stageRange[stage]); + step.V32 = output.V32; + step.V33 = output.V33; + step.V34 = output.V34; + step.V35 = output.V35; + step.V36 = output.V36; + step.V37 = output.V37; + step.V38 = output.V38; + step.V39 = output.V39; + step.V40 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V41 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V42 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V43 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V44 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V45 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V46 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V47 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V48 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V47, cospi[32], output.V48, cosBit); + step.V49 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V46, cospi[32], output.V49, cosBit); + step.V50 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V45, cospi[32], output.V50, cosBit); + step.V51 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V44, cospi[32], output.V51, cosBit); + step.V52 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V43, cospi[32], output.V52, cosBit); + step.V53 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V42, cospi[32], output.V53, cosBit); + step.V54 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V41, cospi[32], output.V54, cosBit); + step.V55 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V40, cospi[32], output.V55, cosBit); + step.V56 = output.V56; + step.V57 = output.V57; + step.V58 = output.V58; + step.V59 = output.V59; + step.V60 = output.V60; + step.V61 = output.V61; + step.V62 = output.V62; + step.V63 = output.V63; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V63, stageRange[stage]); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V62, stageRange[stage]); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V61, stageRange[stage]); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V60, stageRange[stage]); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V59, stageRange[stage]); + output.V5 = Av1Transform1dMath.Clamp(step.V5 + step.V58, stageRange[stage]); + output.V6 = Av1Transform1dMath.Clamp(step.V6 + step.V57, stageRange[stage]); + output.V7 = Av1Transform1dMath.Clamp(step.V7 + step.V56, stageRange[stage]); + output.V8 = Av1Transform1dMath.Clamp(step.V8 + step.V55, stageRange[stage]); + output.V9 = Av1Transform1dMath.Clamp(step.V9 + step.V54, stageRange[stage]); + output.V10 = Av1Transform1dMath.Clamp(step.V10 + step.V53, stageRange[stage]); + output.V11 = Av1Transform1dMath.Clamp(step.V11 + step.V52, stageRange[stage]); + output.V12 = Av1Transform1dMath.Clamp(step.V12 + step.V51, stageRange[stage]); + output.V13 = Av1Transform1dMath.Clamp(step.V13 + step.V50, stageRange[stage]); + output.V14 = Av1Transform1dMath.Clamp(step.V14 + step.V49, stageRange[stage]); + output.V15 = Av1Transform1dMath.Clamp(step.V15 + step.V48, stageRange[stage]); + output.V16 = Av1Transform1dMath.Clamp(step.V16 + step.V47, stageRange[stage]); + output.V17 = Av1Transform1dMath.Clamp(step.V17 + step.V46, stageRange[stage]); + output.V18 = Av1Transform1dMath.Clamp(step.V18 + step.V45, stageRange[stage]); + output.V19 = Av1Transform1dMath.Clamp(step.V19 + step.V44, stageRange[stage]); + output.V20 = Av1Transform1dMath.Clamp(step.V20 + step.V43, stageRange[stage]); + output.V21 = Av1Transform1dMath.Clamp(step.V21 + step.V42, stageRange[stage]); + output.V22 = Av1Transform1dMath.Clamp(step.V22 + step.V41, stageRange[stage]); + output.V23 = Av1Transform1dMath.Clamp(step.V23 + step.V40, stageRange[stage]); + output.V24 = Av1Transform1dMath.Clamp(step.V24 + step.V39, stageRange[stage]); + output.V25 = Av1Transform1dMath.Clamp(step.V25 + step.V38, stageRange[stage]); + output.V26 = Av1Transform1dMath.Clamp(step.V26 + step.V37, stageRange[stage]); + output.V27 = Av1Transform1dMath.Clamp(step.V27 + step.V36, stageRange[stage]); + output.V28 = Av1Transform1dMath.Clamp(step.V28 + step.V35, stageRange[stage]); + output.V29 = Av1Transform1dMath.Clamp(step.V29 + step.V34, stageRange[stage]); + output.V30 = Av1Transform1dMath.Clamp(step.V30 + step.V33, stageRange[stage]); + output.V31 = Av1Transform1dMath.Clamp(step.V31 + step.V32, stageRange[stage]); + output.V32 = Av1Transform1dMath.Clamp(step.V31 - step.V32, stageRange[stage]); + output.V33 = Av1Transform1dMath.Clamp(step.V30 - step.V33, stageRange[stage]); + output.V34 = Av1Transform1dMath.Clamp(step.V29 - step.V34, stageRange[stage]); + output.V35 = Av1Transform1dMath.Clamp(step.V28 - step.V35, stageRange[stage]); + output.V36 = Av1Transform1dMath.Clamp(step.V27 - step.V36, stageRange[stage]); + output.V37 = Av1Transform1dMath.Clamp(step.V26 - step.V37, stageRange[stage]); + output.V38 = Av1Transform1dMath.Clamp(step.V25 - step.V38, stageRange[stage]); + output.V39 = Av1Transform1dMath.Clamp(step.V24 - step.V39, stageRange[stage]); + output.V40 = Av1Transform1dMath.Clamp(step.V23 - step.V40, stageRange[stage]); + output.V41 = Av1Transform1dMath.Clamp(step.V22 - step.V41, stageRange[stage]); + output.V42 = Av1Transform1dMath.Clamp(step.V21 - step.V42, stageRange[stage]); + output.V43 = Av1Transform1dMath.Clamp(step.V20 - step.V43, stageRange[stage]); + output.V44 = Av1Transform1dMath.Clamp(step.V19 - step.V44, stageRange[stage]); + output.V45 = Av1Transform1dMath.Clamp(step.V18 - step.V45, stageRange[stage]); + output.V46 = Av1Transform1dMath.Clamp(step.V17 - step.V46, stageRange[stage]); + output.V47 = Av1Transform1dMath.Clamp(step.V16 - step.V47, stageRange[stage]); + output.V48 = Av1Transform1dMath.Clamp(step.V15 - step.V48, stageRange[stage]); + output.V49 = Av1Transform1dMath.Clamp(step.V14 - step.V49, stageRange[stage]); + output.V50 = Av1Transform1dMath.Clamp(step.V13 - step.V50, stageRange[stage]); + output.V51 = Av1Transform1dMath.Clamp(step.V12 - step.V51, stageRange[stage]); + output.V52 = Av1Transform1dMath.Clamp(step.V11 - step.V52, stageRange[stage]); + output.V53 = Av1Transform1dMath.Clamp(step.V10 - step.V53, stageRange[stage]); + output.V54 = Av1Transform1dMath.Clamp(step.V9 - step.V54, stageRange[stage]); + output.V55 = Av1Transform1dMath.Clamp(step.V8 - step.V55, stageRange[stage]); + output.V56 = Av1Transform1dMath.Clamp(step.V7 - step.V56, stageRange[stage]); + output.V57 = Av1Transform1dMath.Clamp(step.V6 - step.V57, stageRange[stage]); + output.V58 = Av1Transform1dMath.Clamp(step.V5 - step.V58, stageRange[stage]); + output.V59 = Av1Transform1dMath.Clamp(step.V4 - step.V59, stageRange[stage]); + output.V60 = Av1Transform1dMath.Clamp(step.V3 - step.V60, stageRange[stage]); + output.V61 = Av1Transform1dMath.Clamp(step.V2 - step.V61, stageRange[stage]); + output.V62 = Av1Transform1dMath.Clamp(step.V1 - step.V62, stageRange[stage]); + output.V63 = Av1Transform1dMath.Clamp(step.V0 - step.V63, stageRange[stage]); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs new file mode 100644 index 000000000..5a1246eba --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs @@ -0,0 +1,232 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the eight-point AV1 inverse discrete cosine transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply +/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes. +/// +internal readonly struct Av1Dct8Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative eight-point AV1 inverse discrete cosine transform. + /// + /// The eight frequency-domain coefficients. + /// The eight spatial-domain residual values. + /// The eight-element stage buffer owned by the containing two-dimensional transform. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[4]; + output[2] = input[2]; + output[3] = input[6]; + output[4] = input[1]; + output[5] = input[5]; + output[6] = input[3]; + output[7] = input[7]; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } + + /// + /// Applies the transform to four independent axes in parallel. + /// + /// The source values for the parallel transform axes. + /// The destination values for the parallel transform axes. + /// The fixed stage storage for the parallel transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output.V0 = input.V0; + output.V1 = input.V4; + output.V2 = input.V2; + output.V3 = input.V6; + output.V4 = input.V1; + output.V5 = input.V5; + output.V6 = input.V3; + output.V7 = input.V7; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step.V0 = output.V0; + step.V1 = output.V1; + step.V2 = output.V2; + step.V3 = output.V3; + step.V4 = Av1Transform1dMath.HalfButterfly(cospi[56], output.V4, -cospi[8], output.V7, cosBit); + step.V5 = Av1Transform1dMath.HalfButterfly(cospi[24], output.V5, -cospi[40], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[40], output.V5, cospi[24], output.V6, cosBit); + step.V7 = Av1Transform1dMath.HalfButterfly(cospi[8], output.V4, cospi[56], output.V7, cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output.V0 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, cospi[32], step.V1, cosBit); + output.V1 = Av1Transform1dMath.HalfButterfly(cospi[32], step.V0, -cospi[32], step.V1, cosBit); + output.V2 = Av1Transform1dMath.HalfButterfly(cospi[48], step.V2, -cospi[16], step.V3, cosBit); + output.V3 = Av1Transform1dMath.HalfButterfly(cospi[16], step.V2, cospi[48], step.V3, cosBit); + output.V4 = Av1Transform1dMath.Clamp(step.V4 + step.V5, range); + output.V5 = Av1Transform1dMath.Clamp(step.V4 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V7 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V6 + step.V7, range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step.V0 = Av1Transform1dMath.Clamp(output.V0 + output.V3, range); + step.V1 = Av1Transform1dMath.Clamp(output.V1 + output.V2, range); + step.V2 = Av1Transform1dMath.Clamp(output.V1 - output.V2, range); + step.V3 = Av1Transform1dMath.Clamp(output.V0 - output.V3, range); + step.V4 = output.V4; + step.V5 = Av1Transform1dMath.HalfButterfly(-cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V6 = Av1Transform1dMath.HalfButterfly(cospi[32], output.V5, cospi[32], output.V6, cosBit); + step.V7 = output.V7; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output.V0 = Av1Transform1dMath.Clamp(step.V0 + step.V7, range); + output.V1 = Av1Transform1dMath.Clamp(step.V1 + step.V6, range); + output.V2 = Av1Transform1dMath.Clamp(step.V2 + step.V5, range); + output.V3 = Av1Transform1dMath.Clamp(step.V3 + step.V4, range); + output.V4 = Av1Transform1dMath.Clamp(step.V3 - step.V4, range); + output.V5 = Av1Transform1dMath.Clamp(step.V2 - step.V5, range); + output.V6 = Av1Transform1dMath.Clamp(step.V1 - step.V6, range); + output.V7 = Av1Transform1dMath.Clamp(step.V0 - step.V7, range); + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs new file mode 100644 index 000000000..1f9a6c448 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs @@ -0,0 +1,81 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the sixteen-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal readonly struct Av1Identity16Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative sixteen-point AV1 inverse identity transform. + /// + /// The sixteen frequency-domain coefficients. + /// The sixteen scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the twice the square-root-of-two fixed-point scale required for 2-D normalization. + for (int i = 0; i < 16; i++) + { + output[i] = Av1Math.RoundShift((long)input[i] * (2 * Av1Transform1dMath.NewSqrt2), Av1Transform1dMath.NewSqrt2Bits); + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and + // rounding sequence, matching libaom without changing the established lower-range SIMD path. + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs new file mode 100644 index 000000000..416aad01b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs @@ -0,0 +1,65 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the thirty-two-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal readonly struct Av1Identity32Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative thirty-two-point AV1 inverse identity transform. + /// + /// The thirty-two frequency-domain coefficients. + /// The thirty-two scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the exact factor-of-four scale required for 2-D normalization. + for (int i = 0; i < 32; i++) + { + output[i] = input[i] * 4; + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs new file mode 100644 index 000000000..367aa4dbf --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs @@ -0,0 +1,81 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the four-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal readonly struct Av1Identity4Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative four-point AV1 inverse identity transform. + /// + /// The four frequency-domain coefficients. + /// The four scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the square-root-of-two fixed-point scale required for 2-D normalization. + for (int i = 0; i < 4; i++) + { + output[i] = Av1Math.RoundShift((long)input[i] * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product. + // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges. + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + if (stageRange[0] >= Av1Transform1dMath.WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + + _ = step; + _ = cosBit; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs new file mode 100644 index 000000000..8988566b2 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs @@ -0,0 +1,65 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; + +/// +/// Defines the eight-point AV1 inverse identity transform operator. +/// +/// +/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local, +/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis. +/// +internal readonly struct Av1Identity8Inverse1dOperator : IAv1Transform1dOperator +{ + /// + /// Applies the normative eight-point AV1 inverse identity transform. + /// + /// The eight frequency-domain coefficients. + /// The eight scaled spatial-domain values. + /// Unused stage storage supplied by the common transform-kernel contract. + /// Unused cosine precision supplied by the common transform-kernel contract. + /// The signed-bit range assigned to the transform output. + public static void Transform(ReadOnlySpan input, Span output, Span step, int cosBit, Av1TransformStageRange stageRange) + { + _ = step; + _ = cosBit; + _ = stageRange; + + // The AV1 identity transform preserves coefficient order while applying the exact factor-of-two scale required for 2-D normalization. + for (int i = 0; i < 8; i++) + { + output[i] = input[i] * 2; + } + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index aca677eac..fc0ba961e 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -5,6 +5,7 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -39,18 +40,18 @@ public class Av1ForwardTransformTests [Fact] public void OneDimensionalOperatorsMatchAnalyticalReference() { - AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); - AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); - AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); - AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); } /// @@ -87,18 +88,18 @@ public class Av1ForwardTransformTests /// private static void AssertOneDimensionalOperators() { - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(32); - AssertOperator(64); - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(4); - AssertOperator(8); - AssertOperator(16); - AssertOperator(32); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); + AssertOperator(64); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(4); + AssertOperator(8); + AssertOperator(16); + AssertOperator(32); } /// @@ -107,7 +108,7 @@ public class Av1ForwardTransformTests /// The transform operator. /// The transform length. private static void AssertOperator(int length) - where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TOperator : struct, IAv1ForwardTransform1dOperator { const int cosBit = 12; @@ -143,7 +144,7 @@ public class Av1ForwardTransformTests /// The analytical transform definition. /// The transform length. private static void AssertOperatorAccuracy(Av1TransformType1d transformType, int length) - where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TOperator : struct, IAv1ForwardTransform1dOperator { const int cosBit = 13; const int testBlockCount = 500; @@ -190,7 +191,7 @@ public class Av1ForwardTransformTests /// The transform length. /// The fixed-point precision of the cosine constants. private static void AssertInt32Operator(int length, int cosBit) - where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TOperator : struct, IAv1ForwardTransform1dOperator where TVector : struct { int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(int); @@ -244,7 +245,7 @@ public class Av1ForwardTransformTests /// The transform length. /// The fixed-point precision of the cosine constants. private static void AssertInt16Operator(int length, int cosBit) - where TOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TOperator : struct, IAv1ForwardTransform1dOperator where TVector : struct { int laneCount = System.Runtime.CompilerServices.Unsafe.SizeOf() / sizeof(short); @@ -367,40 +368,40 @@ public class Av1ForwardTransformTests switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchReferenceRow(input, stride, output, ref config); + DispatchReferenceRow(input, stride, output, ref config); break; } } @@ -414,45 +415,45 @@ public class Av1ForwardTransformTests /// The destination reference coefficients. /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceRow(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TColumnOperator : struct, IAv1ForwardTransform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct32: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Dct64: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Adst16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity4: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity8: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity16: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; case Av1TransformFunctionType.Identity32: - TransformReference(input, stride, output, ref config); + TransformReference(input, stride, output, ref config); break; } } @@ -471,8 +472,8 @@ public class Av1ForwardTransformTests int stride, Span output, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator - where TRowOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator + where TColumnOperator : struct, IAv1ForwardTransform1dOperator + where TRowOperator : struct, IAv1ForwardTransform1dOperator { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index b255cd041..15b0d98f9 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -4,6 +4,8 @@ using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -53,11 +55,11 @@ public class Av1InverseTransformTests /// private static void AssertDctOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); - AssertOperatorParity(64); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); + AssertOperatorParity(32); + AssertOperatorParity(64); } /// @@ -65,9 +67,9 @@ public class Av1InverseTransformTests /// private static void AssertAdstOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); } /// @@ -75,10 +77,10 @@ public class Av1InverseTransformTests /// private static void AssertIdentityOperatorParity() { - AssertOperatorParity(4); - AssertOperatorParity(8); - AssertOperatorParity(16); - AssertOperatorParity(32); + AssertOperatorParity(4); + AssertOperatorParity(8); + AssertOperatorParity(16); + AssertOperatorParity(32); } /// @@ -108,14 +110,14 @@ public class Av1InverseTransformTests Av1TransformVector> adstOutput256 = default; Av1TransformVector> adstStep256 = default; - Av1InverseTransformer.Adst4Operator.Transform( + Av1Adst4Inverse1dOperator.Transform( ref adstInput128, ref adstOutput128, ref adstStep128, cosBit, stageRange); - Av1InverseTransformer.Adst4Operator.Transform( + Av1Adst4Inverse1dOperator.Transform( ref adstInput256, ref adstOutput256, ref adstStep256, @@ -148,7 +150,7 @@ public class Av1InverseTransformTests 262_143, -262_144); - AssertWidenedIdentityOperator( + AssertWidenedIdentityOperator( 4, identityInput128, Vector128.Create(741_503, -741_504, 741_501, -741_503), @@ -156,7 +158,7 @@ public class Av1InverseTransformTests Vector256.Create(741_503, -741_504, 741_501, -741_503, 524_322, -524_323, 370_751, -370_752), stageRange); - AssertWidenedIdentityOperator( + AssertWidenedIdentityOperator( 16, identityInput128, Vector128.Create(1_483_005, -1_483_008, 1_483_002, -1_483_005), @@ -182,7 +184,7 @@ public class Av1InverseTransformTests Vector256 input256, Vector256 expected256, Av1TransformStageRange stageRange) - where TOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TOperator : struct, IAv1Transform1dOperator { const int cosBit = 12; Av1TransformVector> values128 = default; @@ -268,18 +270,18 @@ public class Av1InverseTransformTests [Fact] public void ForwardAndInverseOperatorPairsReconstructTheirInput() { - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size8x8, 2, 2); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size16x16, 3, 3); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size32x32, 4, 4); - AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size64x64, 5, 5); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size8x8, 2, 2); - AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size16x16, 3, 3); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size4x4, 1, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size8x8, 2, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size16x16, 3, 1); - AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size8x8, 2, 2); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size16x16, 3, 3); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size32x32, 4, 4); + AssertRoundTrip(Av1TransformType.DctDct, Av1TransformSize.Size64x64, 5, 5); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size8x8, 2, 2); + AssertRoundTrip(Av1TransformType.AdstAdst, Av1TransformSize.Size16x16, 3, 3); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size4x4, 1, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size8x8, 2, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size16x16, 3, 1); + AssertRoundTrip(Av1TransformType.Identity, Av1TransformSize.Size32x32, 4, 1); } /// @@ -596,7 +598,7 @@ public class Av1InverseTransformTests /// The inverse transform operator. /// The transform length. private static void AssertOperatorParity(int length) - where TOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TOperator : struct, IAv1Transform1dOperator { const int cosBit = 12; Av1TransformStageRange stageRange = default; @@ -670,8 +672,8 @@ public class Av1InverseTransformTests /// The power-of-two scale applied by the operator pair. /// The maximum permitted reconstruction error. private static void AssertRoundTrip(Av1TransformType transformType, Av1TransformSize transformSize, int scaleLog2, int allowedError) - where TForwardOperator : struct, Av1ForwardTransformer.IAv1ForwardTransform1dOperator - where TInverseOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TForwardOperator : struct, IAv1ForwardTransform1dOperator + where TInverseOperator : struct, IAv1Transform1dOperator { const int bitDepth = 10; const int testBlockCount = 30; @@ -730,40 +732,40 @@ public class Av1InverseTransformTests switch (config.TransformFunctionTypeColumn) { case Av1TransformFunctionType.Dct4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct64: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity4: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity8: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity16: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity32: - DispatchRow(transformType, transformSize, bitDepth, ref config); + DispatchRow(transformType, transformSize, bitDepth, ref config); break; default: Assert.Fail($"Unexpected column function {config.TransformFunctionTypeColumn} for {transformType} {transformSize}."); @@ -784,45 +786,45 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator { switch (config.TransformFunctionTypeRow) { case Av1TransformFunctionType.Dct4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct32: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Dct64: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Adst16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity4: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity8: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity16: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; case Av1TransformFunctionType.Identity32: - AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); + AssertTransform2dParity(transformType, transformSize, bitDepth, ref config); break; default: Assert.Fail($"Unexpected row function {config.TransformFunctionTypeRow} for {transformType} {transformSize}."); @@ -844,8 +846,8 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator - where TRowOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -895,8 +897,8 @@ public class Av1InverseTransformTests int[] coefficients, Av1TransformSize transformSize, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator - where TRowOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { const int bitDepth = 8; int width = transformSize.GetWidth(); @@ -921,10 +923,10 @@ public class Av1InverseTransformTests Array.Fill(scalar, byte.MaxValue); Array.Fill(vector128, byte.MaxValue); - Av1InverseTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); - Av1InverseTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); @@ -935,7 +937,7 @@ public class Av1InverseTransformTests int[] vector256Workspace = new int[workspaceLength]; Array.Fill(vector256, byte.MaxValue); - Av1InverseTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); Assert.Equal(scalar, vector256); @@ -956,8 +958,8 @@ public class Av1InverseTransformTests Av1TransformSize transformSize, int bitDepth, ref Av1Transform2dFlipConfiguration config) - where TColumnOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator - where TRowOperator : struct, Av1InverseTransformer.IAv1InverseTransform1dOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator { int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); @@ -982,10 +984,10 @@ public class Av1InverseTransformTests Array.Fill(scalar, short.MinValue); Array.Fill(vector128, short.MinValue); - Av1InverseTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth); - Av1InverseTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth); Assert.Equal(scalar, vector128); @@ -996,7 +998,7 @@ public class Av1InverseTransformTests int[] vector256Workspace = new int[workspaceLength]; Array.Fill(vector256, short.MinValue); - Av1InverseTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( + Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>( coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth); Assert.Equal(scalar, vector256);