From 20aa6515589ddcea531a2cb2d16afa5be40bb389 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Thu, 27 Aug 2026 04:01:40 +1000 Subject: [PATCH] Complete libaom AV1 forward transform port --- HEIF_IMPLEMENTATION_PLAN.md | 9 +- .../Av1/Transform/Av1ForwardTransformer.cs | 199 ++++-- .../Forward/Av1Adst16Forward1dOperator.cs | 10 +- .../Forward/Av1Adst4Forward1dOperator.cs | 10 +- .../Forward/Av1Adst8Forward1dOperator.cs | 10 +- .../Forward/Av1Dct16Forward1dOperator.cs | 10 +- .../Forward/Av1Dct32Forward1dOperator.cs | 10 +- .../Forward/Av1Dct4Forward1dOperator.cs | 10 +- .../Forward/Av1Dct64Forward1dOperator.cs | 10 +- .../Forward/Av1Dct8Forward1dOperator.cs | 10 +- .../Av1ForwardTransformOperations.Adst.cs | 337 +++++----- .../Av1ForwardTransformOperations.Dct16.cs | 292 ++++++--- .../Av1ForwardTransformOperations.Dct32.cs | 417 ++++++------ .../Av1ForwardTransformOperations.Dct4.cs | 165 ++++- .../Av1ForwardTransformOperations.Dct64.cs | 600 +++++++----------- .../Av1ForwardTransformOperations.Dct8.cs | 137 ++-- .../Av1ForwardTransformOperations.Identity.cs | 122 ++-- .../Forward/Av1Identity16Forward1dOperator.cs | 10 +- .../Forward/Av1Identity32Forward1dOperator.cs | 10 +- .../Forward/Av1Identity4Forward1dOperator.cs | 10 +- .../Forward/Av1Identity8Forward1dOperator.cs | 10 +- .../Forward/IAv1ForwardTransform1dOperator.cs | 16 +- .../Heif/Av1/Av1ForwardTransformTests.cs | 140 +++- .../Heif/Av1/Av1InverseTransformTests.cs | 6 +- 24 files changed, 1513 insertions(+), 1047 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index ea221e2f8..334cdc63f 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -29,7 +29,7 @@ Checkboxes may be marked complete only when the implementation and the verificat ## Delivery dashboard -Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `2bd81bb7c`, including the completed AV1 transform architecture checkpoint. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. +Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `20df9115d`, including the completed AV1 transform architecture checkpoint. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. Status meanings: @@ -38,7 +38,7 @@ Status meanings: - **Not started:** supporting primitives may exist, but the production format path is absent. - **Current:** the only work item that should be advanced before taking the next queued item. -Current development stage: **Stage 3 — complete AV1 still-image decoding.** The transform checkpoint is closed: forward transforms use one libaom-shaped operator architecture across scalar, `Vector128`, `Vector256`, and `Vector512`, inverse production traversal uses the verified scalar, `Vector128`, and `Vector256` tiers, and implementation-mechanic type and file suffixes have been removed. Neither AV1 nor HEVC production encoding is implemented. +Current development stage: **Stage 3 — complete AV1 still-image decoding.** The transform checkpoint is closed: forward transforms use one libaom-shaped SIMD-first operator architecture across `Vector512`, `Vector256`, and `Vector128`, with scalar fallback; inverse production traversal uses the verified `Vector256` and `Vector128` tiers with scalar fallback; and implementation-mechanic type and file suffixes have been removed. Neither AV1 nor HEVC production encoding is implemented. Immediate checkpoint: **remove every remaining valid AV1 still-image unsupported branch and prove the complete decode matrix.** Each syntax tool must be implemented through the established SIMD-first architecture with scalar fallback and verified with independent AVIF/libaom evidence across supported bit depths, chroma layouts, filters, grain, and color signaling. @@ -119,15 +119,16 @@ Performance, allocation, documentation, and independent test work are part of ea - [x] Select transform type, size, bit depth, and ISA once at the 2-D block boundary rather than dispatching through an interface for every row and column. - [x] Port the DCT4/8/16/32/64, ADST4/8/16, and identity4/8/16/32 stage networks from the pinned libaom scalar and Highway sources into one static-generic operator architecture. - [x] Implement paired add/subtract and whole-butterfly primitives for scalar, `Vector128`, `Vector256`, and `Vector512` values, including saturated packed arithmetic and shared widening work at each supported SIMD width. - - [x] Implement the complete libaom two-dimensional load, flip, shift, axis-transform, transpose, rectangle-normalization, promotion, and 64-point coefficient-retention pipeline without per-block allocation. + - [x] Implement the complete libaom two-dimensional load, flip, shift, axis-transform, transpose, rectangle-normalization, promotion, and 64-point coefficient-retention pipeline without per-block allocation. The axis driver passes independent input and output strides directly to the two fixed stage buffers, and the 64x64 row transform retires only the retained 32 coefficients without a transform-sized copy pass. - [x] Port the applicable libaom bulk inverse-transform kernels using the same tables, rounding, saturation, and clipping rules as the scalar oracle. - [x] Use normal ImageSharp descending-width dispatch and require the actual packed arithmetic ISA when selecting packed `Vector512` traversal. - [x] Document scratch ownership, stage-buffer alternation, fixed-point rounding, lane layout, transposition, and scalar fallback decisions at their implementation points. - [x] Remove the separate SIMD files, width-specific forward contracts, and sample-representation suffixes so each transform operator owns one behavior model. - [x] Verify every 1-D operator representation and every valid 2-D size/type/bit-depth combination through `FeatureTestRunner` with normal hardware, AVX-512 disabled, AVX disabled, and all hardware intrinsics disabled. - The focused Release run passes all 511 forward and inverse transform cases. The suite covers DCT, ADST, and identity operators, packed overflow-sensitive inputs, padded 2-D input strides, all valid transform configurations, 8/10/12-bit dispatch, inverse reconstruction, and the zero-allocation block contract. + - [x] Verify DCT4/8/16/32/64, ADST4/8/16, and identity4/8/16/32 independently against the analytical transform oracle and coefficient-error bound used by the pinned libaom forward-transform tests. - [x] Benchmark the production 32x32 DCT path after the complete paired stage port with preferred 256-bit and 512-bit widths. - - On the measured .NET 10 AVX-512 host, the 8-bit path measured 1.334 microseconds at 256 bits and 1.349 microseconds at 512 bits. The 12-bit path measured 2.977 microseconds at 256 bits and 2.008 microseconds at 512 bits. BenchmarkDotNet reported no managed allocation for any measured path, so production retains the normal runtime-selected width instead of a transform-type or bit-depth patch table. + - On the measured .NET 10 AVX-512 host, the packed 8-bit path measured 856.6 nanoseconds at 256 bits and 639.8 nanoseconds at 512 bits, making AVX-512BW 25.3% faster. The expanded 12-bit path measured 976.2 nanoseconds at 256 bits and 1019.8 nanoseconds at 512 bits, making the runtime-preferred 256-bit width 4.5% faster. BenchmarkDotNet reported no managed allocation. Production therefore follows Highway's AVX-512BW capability dispatch for packed stages and the runtime-preferred width for expanded stages. - [ ] **Queued:** restore bounded animated HEIC and AVIF image-sequence scope, including the required image-level and per-frame metadata contracts, without introducing unrelated ISO BMFF surfaces. - [x] Reconcile the top-level still-image-only scope with the required animated HEIC and AVIF completion boundary before sequence implementation begins. - [x] Define the ImageSharp image-level sequence metadata and per-frame metadata contracts, including observable timing, repetition, color, alpha, orientation, and profile behavior. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs index de471deba..5ff9464ea 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs @@ -42,6 +42,12 @@ internal static class Av1ForwardTransformer /// /// Selects the concrete column operator for a transform block. /// + /// The spatial residual samples. + /// The destination transform coefficients. + /// The number of input samples between rows. + /// The source sample bit depth. + /// The resolved transform functions, shifts, and axis orientation. + /// The reusable transform workspace. private static void DispatchColumn( Span input, Span coefficients, @@ -96,6 +102,13 @@ internal static class Av1ForwardTransformer /// /// Selects the concrete row operator after the column operator has been specialized. /// + /// The column transform operator selected for the block. + /// The spatial residual samples. + /// The destination transform coefficients. + /// The number of input samples between rows. + /// The source sample bit depth. + /// The resolved transform functions, shifts, and axis orientation. + /// The reusable transform workspace. private static void DispatchRow( Span input, Span coefficients, @@ -151,6 +164,14 @@ internal static class Av1ForwardTransformer /// /// Applies the specialized operator pair using the sample representation selected for the coded bit depth. /// + /// The column transform operator. + /// The row transform operator. + /// The spatial residual samples. + /// The destination transform coefficients. + /// The number of input samples between rows. + /// The source sample bit depth. + /// The resolved transform functions, shifts, and axis orientation. + /// The reusable transform workspace. private static void Transform2d( Span input, Span coefficients, @@ -175,6 +196,13 @@ internal static class Av1ForwardTransformer /// /// Applies the libaom Int16 stage pipeline used for eight-bit residuals. /// + /// The column transform operator. + /// The row transform operator. + /// The spatial residual samples. + /// The destination transform coefficients. + /// The number of input samples between rows. + /// The resolved transform functions, shifts, and axis orientation. + /// The reusable transform workspace. private static void TransformPacked( Span input, Span coefficients, @@ -186,7 +214,13 @@ internal static class Av1ForwardTransformer { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); - int blockLaneCount = Avx512BW.IsSupported ? Vector512.Count : Avx2.IsSupported ? Vector256.Count : Vector128.Count; + + // Packed short stages halve the arithmetic width and AVX-512BW doubles their lane count. Highway selects + // this representation by ISA capability, independently of the runtime preference used for generic vectors. + int blockLaneCount = Avx512BW.IsSupported + ? Vector512.Count + : Avx2.IsSupported ? Vector256.Count : Vector128.Count; + int blockWidth = Math.Max(width, blockLaneCount); int blockHeight = Math.Max(height, blockLaneCount); int blockArea = blockWidth * blockHeight; @@ -197,7 +231,7 @@ internal static class Av1ForwardTransformer ref short buffer0Base = ref MemoryMarshal.GetReference(buffer0); LoadPacked(input, stride, ref buffer0Base, blockWidth, width, height, config.Shift0, config.FlipUpsideDown, config.FlipLeftToRight); - TransformPackedAxis(buffer0, height, width, blockWidth, config.CosBitColumn, workspace); + TransformPackedAxis(buffer0, width, blockWidth, blockWidth, config.CosBitColumn, workspace); int retainedHeight = Math.Min(height, 32); int retainedWidth = Math.Min(width, 32); @@ -220,12 +254,14 @@ internal static class Av1ForwardTransformer -config.Shift1, scratch); - TransformExpandedAxis(buffer1, width, retainedHeight, blockHeight, config.CosBitRow, workspace); + int rowOutputStride = width == 64 && height == 64 ? 32 : blockHeight; + + TransformExpandedAxis(buffer1, retainedHeight, blockHeight, rowOutputStride, config.CosBitRow, workspace); ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); TransposeExpanded( ref buffer1Base, - blockHeight, + rowOutputStride, ref coefficientBase, retainedWidth, retainedHeight, @@ -253,7 +289,7 @@ internal static class Av1ForwardTransformer false, packedScratch); - TransformPackedAxis(buffer1Packed, width, height, blockHeight, config.CosBitRow, workspace); + TransformPackedAxis(buffer1Packed, height, blockHeight, blockHeight, config.CosBitRow, workspace); // The second transform produces horizontal frequency in rows and vertical frequency in lanes. Transposing // once more adapts libaom's native layout to the row-major coefficient contract used by ImageSharp. @@ -274,6 +310,13 @@ internal static class Av1ForwardTransformer /// /// Applies the libaom Int32 stage pipeline used for high-bit-depth residuals and scalar fallback. /// + /// The column transform operator. + /// The row transform operator. + /// The spatial residual samples. + /// The destination transform coefficients. + /// The number of input samples between rows. + /// The resolved transform functions, shifts, and axis orientation. + /// The reusable transform workspace. private static void TransformExpanded( Span input, Span coefficients, @@ -285,7 +328,10 @@ internal static class Av1ForwardTransformer { int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); - int blockLaneCount = Vector512.IsHardwareAccelerated ? Vector512.Count : Vector256.IsHardwareAccelerated ? Vector256.Count : Vector128.IsHardwareAccelerated ? Vector128.Count : 1; + int blockLaneCount = Vector512.IsHardwareAccelerated + ? Vector512.Count + : Vector256.IsHardwareAccelerated ? Vector256.Count : Vector128.IsHardwareAccelerated ? Vector128.Count : 1; + int blockWidth = Math.Max(width, blockLaneCount); int blockHeight = Math.Max(height, blockLaneCount); int blockArea = blockWidth * blockHeight; @@ -297,7 +343,7 @@ internal static class Av1ForwardTransformer ref int buffer1Base = ref MemoryMarshal.GetReference(buffer1); LoadExpanded(input, stride, ref buffer0Base, blockWidth, width, height, config.Shift0, config.FlipUpsideDown, config.FlipLeftToRight); - TransformExpandedAxis(buffer0, height, width, blockWidth, config.CosBitColumn, workspace); + TransformExpandedAxis(buffer0, width, blockWidth, blockWidth, config.CosBitColumn, workspace); TransposeExpanded( ref buffer0Base, @@ -312,13 +358,15 @@ internal static class Av1ForwardTransformer int retainedHeight = Math.Min(height, 32); int retainedWidth = Math.Min(width, 32); - TransformExpandedAxis(buffer1, width, retainedHeight, blockHeight, config.CosBitRow, workspace); + int rowOutputStride = width == 64 && height == 64 ? 32 : blockHeight; + + TransformExpandedAxis(buffer1, retainedHeight, blockHeight, rowOutputStride, config.CosBitRow, workspace); bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; ref int coefficientBase = ref MemoryMarshal.GetReference(coefficients); TransposeExpanded( ref buffer1Base, - blockHeight, + rowOutputStride, ref coefficientBase, retainedWidth, retainedHeight, @@ -331,6 +379,15 @@ internal static class Av1ForwardTransformer /// /// Loads, flips, and scales one eight-bit residual block into packed transform storage. /// + /// The spatial residual samples. + /// The number of input samples between rows. + /// The first value in the packed transform block. + /// The number of packed values between destination rows. + /// The transform-block width. + /// The transform-block height. + /// The initial transform scaling shift. + /// Whether to reverse the input row order. + /// Whether to reverse the samples within each row. private static void LoadPacked( Span input, uint inputStride, @@ -403,6 +460,15 @@ internal static class Av1ForwardTransformer /// /// Loads, flips, widens, and scales one residual block into signed thirty-two-bit transform storage. /// + /// The spatial residual samples. + /// The number of input samples between rows. + /// The first value in the expanded transform block. + /// The number of expanded values between destination rows. + /// The transform-block width. + /// The transform-block height. + /// The initial transform scaling shift. + /// Whether to reverse the input row order. + /// Whether to reverse the samples within each row. private static void LoadExpanded( Span input, uint inputStride, @@ -473,71 +539,94 @@ internal static class Av1ForwardTransformer /// /// Applies one packed transform axis using the widest efficient lane count available for the block. /// + /// The transform operator applied to each independent axis. + /// The packed transform block. + /// The number of independent axes. + /// The number of packed values between input positions. + /// The number of packed values between output positions. + /// The fixed-point precision of the cosine constants. + /// The reusable transform-stage workspace. private static void TransformPackedAxis( Span buffer, - int transformLength, int transformCount, - int stride, + int inputStride, + int outputStride, int cosBit, Span workspace) where TOperator : struct, IAv1ForwardTransform1dOperator { if (Avx512BW.IsSupported && transformCount >= Vector512.Count) { - TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } if (Avx2.IsSupported && transformCount >= Vector256.Count) { - TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } - TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); } /// /// Applies one signed thirty-two-bit transform axis using the widest efficient lane count available for the block. /// + /// The transform operator applied to each independent axis. + /// The expanded transform block. + /// The number of independent axes. + /// The number of expanded values between input positions. + /// The number of expanded values between output positions. + /// The fixed-point precision of the cosine constants. + /// The reusable transform-stage workspace. private static void TransformExpandedAxis( Span buffer, - int transformLength, int transformCount, - int stride, + int inputStride, + int outputStride, int cosBit, Span workspace) where TOperator : struct, IAv1ForwardTransform1dOperator { if (Vector512.IsHardwareAccelerated && transformCount >= Vector512.Count) { - TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } if (Vector256.IsHardwareAccelerated && transformCount >= Vector256.Count) { - TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } if (Vector128.IsHardwareAccelerated && transformCount >= Vector128.Count) { - TransformAxis>(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis>(buffer, transformCount, inputStride, outputStride, cosBit, workspace); return; } - TransformAxis(buffer, transformLength, transformCount, stride, cosBit, workspace); + TransformAxis(buffer, transformCount, inputStride, outputStride, cosBit, workspace); } /// /// Applies one transform stage network to independent axes held in scalar or SIMD lanes. /// + /// The transform operator applied to each independent axis. + /// The scalar storage element. + /// The scalar or SIMD value containing independent transform axes. + /// The transform block. + /// The number of independent axes. + /// The number of storage elements between input positions. + /// The number of storage elements between output positions. + /// The fixed-point precision of the cosine constants. + /// The reusable transform-stage workspace. private static void TransformAxis( Span buffer, - int transformLength, int transformCount, - int stride, + int inputStride, + int outputStride, int cosBit, Span workspace) where TOperator : struct, IAv1ForwardTransform1dOperator @@ -547,34 +636,36 @@ internal static class Av1ForwardTransformer int vectorByteLength = Unsafe.SizeOf>(); int laneCount = Unsafe.SizeOf() / Unsafe.SizeOf(); ref byte workspaceBase = ref Unsafe.As(ref MemoryMarshal.GetReference(workspace)); - ref Av1TransformVector input = ref Unsafe.As>(ref workspaceBase); - ref Av1TransformVector output = ref Unsafe.As>(ref Unsafe.Add(ref workspaceBase, vectorByteLength)); - ref Av1TransformVector step = ref Unsafe.As>(ref Unsafe.Add(ref workspaceBase, 2 * vectorByteLength)); + ref Av1TransformVector buffer0 = ref Unsafe.As>(ref workspaceBase); + ref Av1TransformVector buffer1 = + ref Unsafe.As>(ref Unsafe.Add(ref workspaceBase, vectorByteLength)); + ref TElement sourceBase = ref MemoryMarshal.GetReference(buffer); + nint inputByteStride = inputStride * Unsafe.SizeOf(); + nint outputByteStride = outputStride * Unsafe.SizeOf(); - // Each lane is an independent row or column. Reusing the three caller-owned vectors for every batch keeps - // the complete 1-D stage network in registers without creating a transform-sized stack frame per axis. + // Each lane is an independent row or column. The operators load from and retire coefficients directly to + // the strided block, matching Highway's two-buffer stage network without a separate input/output copy pass. for (int batch = 0; batch < transformCount; batch += laneCount) { - for (int index = 0; index < transformLength; index++) - { - ref TElement source = ref Unsafe.Add(ref sourceBase, (index * stride) + batch); - input[index] = Unsafe.ReadUnaligned(ref Unsafe.As(ref source)); - } - - TOperator.Transform(ref input, ref output, ref step, cosBit); + ref byte values = ref Unsafe.As(ref Unsafe.Add(ref sourceBase, batch)); - for (int index = 0; index < transformLength; index++) - { - ref TElement destination = ref Unsafe.Add(ref sourceBase, (index * stride) + batch); - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), output[index]); - } + TOperator.Transform(ref values, inputByteStride, outputByteStride, ref buffer0, ref buffer1, cosBit); } } /// /// Transposes packed transform storage while applying an AV1 pipeline shift and optional rectangle scaling. /// + /// The first value in the source block. + /// The number of packed values between source rows. + /// The first value in the destination block. + /// The number of packed values between destination rows. + /// The number of source columns. + /// The number of source rows. + /// The signed AV1 scaling shift. + /// Whether to apply the square-root-of-two rectangle normalization. + /// The reusable transpose workspace. private static void TransposePacked( ref short source, int sourceStride, @@ -634,6 +725,14 @@ internal static class Av1ForwardTransformer /// /// Promotes and transposes the large packed layouts at the same axis boundary as libaom. /// + /// The first packed value in the source block. + /// The number of packed values between source rows. + /// The first expanded value in the destination block. + /// The number of expanded values between destination rows. + /// The number of source columns. + /// The number of source rows. + /// The signed AV1 scaling shift. + /// The reusable conversion and transpose workspace. private static void TransposeAndPromote( ref short source, int sourceStride, @@ -694,6 +793,15 @@ internal static class Av1ForwardTransformer /// /// Transposes signed thirty-two-bit transform storage while applying the configured terminal operations. /// + /// The first value in the source block. + /// The number of expanded values between source rows. + /// The first value in the destination block. + /// The number of expanded values between destination rows. + /// The number of source columns. + /// The number of source rows. + /// The signed AV1 scaling shift. + /// Whether to apply the square-root-of-two rectangle normalization. + /// The reusable transpose workspace. private static void TransposeExpanded( ref int source, int sourceStride, @@ -706,7 +814,10 @@ internal static class Av1ForwardTransformer Span scratch) { bool useVector512 = Vector512.IsHardwareAccelerated && Math.Min(sourceWidth, sourceHeight) >= 16; - int tileSize = useVector512 ? 16 : Vector256.IsHardwareAccelerated && Math.Min(sourceWidth, sourceHeight) >= 8 ? 8 : Vector128.IsHardwareAccelerated ? 4 : 1; + int tileSize = useVector512 + ? 16 + : Vector256.IsHardwareAccelerated && Math.Min(sourceWidth, sourceHeight) >= 8 ? 8 : Vector128.IsHardwareAccelerated ? 4 : 1; + Span transposeScratch = MemoryMarshal.Cast(scratch); for (int row = 0; row < sourceHeight; row += tileSize) @@ -761,6 +872,10 @@ internal static class Av1ForwardTransformer /// /// Widens the completed packed coefficient matrix into its external signed thirty-two-bit representation. /// + /// The first packed transform coefficient. + /// The coefficient matrix width. + /// The coefficient matrix height. + /// The destination signed thirty-two-bit coefficients. private static void StorePacked(ref short source, int width, int height, Span destination) { ref int destinationBase = ref MemoryMarshal.GetReference(destination); @@ -803,7 +918,9 @@ internal static class Av1ForwardTransformer if (column < width) { - Vector128 value = Vector128.WidenLower(Vector128.Create(Unsafe.As(ref Unsafe.Add(ref sourceRow, column)), 0UL).AsInt16()); + Vector128 value = Vector128.WidenLower( + Vector128.Create(Unsafe.As(ref Unsafe.Add(ref sourceRow, column)), 0UL).AsInt16()); + value.StoreUnsafe(ref destinationRow, (nuint)column); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs index 589f33823..3c354fd6c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Adst16Forward1dOperator : IAv1ForwardTransform1dOper { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Adst16(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Adst16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs index cf38e8a31..ec561c746 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Adst4Forward1dOperator : IAv1ForwardTransform1dOpera { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Adst4(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Adst4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs index e89715f4d..1d7e0a424 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Adst8Forward1dOperator : IAv1ForwardTransform1dOpera { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Adst8(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Adst8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs index e4f768a7f..e1b447b15 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Dct16Forward1dOperator : IAv1ForwardTransform1dOpera { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Dct16(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Dct16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs index 4efb9f27e..5749ab40e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Dct32Forward1dOperator : IAv1ForwardTransform1dOpera { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Dct32(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Dct32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs index d45796ddd..bef5993f3 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Dct4Forward1dOperator : IAv1ForwardTransform1dOperat { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Dct4(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Dct4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs index 963e140f9..ca12aa88a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Dct64Forward1dOperator : IAv1ForwardTransform1dOpera { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Dct64(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Dct64(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs index 3d002d6ac..8fc637085 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Dct8Forward1dOperator : IAv1ForwardTransform1dOperat { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Dct8(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Dct8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs index c194fcbf7..9473e7663 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Adst.cs @@ -8,44 +8,63 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// internal static partial class Av1ForwardTransformOperations { + /// + /// Gets the fixed eight-point ADST coefficient permutation. + /// + private static ReadOnlySpan Adst8OutputOrder => [1, 6, 3, 4, 5, 2, 7, 0]; + + /// + /// Gets the first cosine index for each final sixteen-point ADST rotation. + /// + private static ReadOnlySpan Adst16FinalWeights => [2, 10, 18, 26, 34, 42, 50, 58]; + + /// + /// Gets the fixed sixteen-point ADST coefficient permutation. + /// + private static ReadOnlySpan Adst16OutputOrder => [1, 14, 3, 12, 5, 10, 7, 8, 9, 6, 11, 4, 13, 2, 15, 0]; + /// /// Applies the four-point forward asymmetric discrete sine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The unused transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The unused first transform-stage buffer. + /// The unused second transform-stage buffer. /// The fixed-point precision of the sine constants. public static void Adst4( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { - _ = step; + _ = buffer0; + _ = buffer1; ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - TValue input0 = input[0]; - TValue input1 = input[1]; - TValue input2 = input[2]; - TValue input3 = input[3]; + TValue input0 = Load(ref values, inputStride, 0); + TValue input1 = Load(ref values, inputStride, 1); + TValue input2 = Load(ref values, inputStride, 2); + TValue input3 = Load(ref values, inputStride, 3); TValue input01 = Av1ForwardTransformArithmetic.Add(input0, input1); - // Highway forms x0 + x1 in the native lane width before widening the products. Retaining that intermediate - // is observable for Int16 overflow and is therefore part of the reference stage network, not an algebraic - // simplification opportunity. - output[0] = Av1ForwardTransformArithmetic.MultiplyAddRound( + // Packed lanes form input0 + input1 before widening, matching Highway's observable saturating arithmetic. + TValue output0 = Av1ForwardTransformArithmetic.MultiplyAddRound( sinpi[1], input0, sinpi[2], input1, sinpi[3], input2, sinpi[4], input3, cosBit, in rounding); - output[1] = Av1ForwardTransformArithmetic.MultiplyAddRound( + + TValue output1 = Av1ForwardTransformArithmetic.MultiplyAddRound( sinpi[3], input01, -sinpi[3], input3, 0, input0, 0, input0, cosBit, in rounding); - output[2] = Av1ForwardTransformArithmetic.MultiplyAddRound( + + TValue output2 = Av1ForwardTransformArithmetic.MultiplyAddRound( sinpi[4], input0, -sinpi[1], input1, -sinpi[3], input2, sinpi[2], input3, cosBit, in rounding); - // The final output is Highway's widened w2 - w0 + 3 * v5 sequence expressed with the same unrounded - // products. All four outputs then share the single normative fixed-point rounding point. - output[3] = Av1ForwardTransformArithmetic.MultiplyAddRound( + // This expression preserves Highway's widened w2 - w0 + 3 * v5 sequence with one rounding point. + TValue output3 = Av1ForwardTransformArithmetic.MultiplyAddRound( sinpi[4] - sinpi[1], input0, -sinpi[1] - sinpi[2], @@ -56,46 +75,52 @@ internal static partial class Av1ForwardTransformOperations input3, cosBit, in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); } /// /// Applies the eight-point forward asymmetric discrete sine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values and first transform-stage buffer. - /// The second transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Adst8( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - // Stage 1 applies the ADST input permutation and signs. The following stages can then use the same adjacent - // butterfly layout across every scalar and SIMD instantiation. - output[0] = input[0]; - output[1] = Av1ForwardTransformArithmetic.Negate(input[7]); - output[2] = Av1ForwardTransformArithmetic.Negate(input[3]); - output[3] = input[4]; - output[4] = Av1ForwardTransformArithmetic.Negate(input[1]); - output[5] = input[6]; - output[6] = input[2]; - output[7] = Av1ForwardTransformArithmetic.Negate(input[5]); + // Stage 1 applies the ADST permutation and signs while the source block is still read-only. + buffer0[0] = Load(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); + buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); + buffer0[3] = Load(ref values, inputStride, 4); + buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); + buffer0[5] = Load(ref values, inputStride, 6); + buffer0[6] = Load(ref values, inputStride, 2); + buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); // Stage 2 rotates the second pair in each four-value group while copying the already aligned pairs. - step[0] = output[0]; - step[1] = output[1]; - Av1ForwardTransformArithmetic.Butterfly( - cospi[32], cospi[32], output[2], output[3], out step[2], out step[3], cosBit, in rounding); - step[4] = output[4]; - step[5] = output[5]; - Av1ForwardTransformArithmetic.Butterfly( - cospi[32], cospi[32], output[6], output[7], out step[6], out step[7], cosBit, in rounding); + buffer1[0] = buffer0[0]; + buffer1[1] = buffer0[1]; + Butterfly(cospi[32], cospi[32], buffer0[2], buffer0[3], ref buffer1, 2, 3, cosBit, in rounding); + buffer1[4] = buffer0[4]; + buffer1[5] = buffer0[5]; + Butterfly(cospi[32], cospi[32], buffer0[6], buffer0[7], ref buffer1, 6, 7, cosBit, in rounding); // Stage 3 combines the rotated and copied pairs into two independent four-value groups. for (int group = 0; group < 8; group += 4) @@ -103,113 +128,95 @@ internal static partial class Av1ForwardTransformOperations for (int i = 0; i < 2; i++) { Av1ForwardTransformArithmetic.AddSubtract( - step[group + i], - step[group + i + 2], - out output[group + i], - out output[group + i + 2]); + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); } } - // Stage 4 rotates the upper group by pi/8 and retains the completed lower group. + // Stage 4 rotates the upper group by pi/8 while the completed lower group passes through unchanged. for (int i = 0; i < 4; i++) { - step[i] = output[i]; + buffer1[i] = buffer0[i]; } - step[4] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[16], output[4], cospi[48], output[5], cosBit, in rounding); - step[5] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[48], output[4], -cospi[16], output[5], cosBit, in rounding); - step[6] = Av1ForwardTransformArithmetic.HalfButterfly( - -cospi[48], output[6], cospi[16], output[7], cosBit, in rounding); - step[7] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[16], output[6], cospi[48], output[7], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[4], cospi[48], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[48], buffer0[4], -cospi[16], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[48], buffer0[6], cospi[16], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[16], buffer0[6], cospi[48], buffer0[7], cosBit, in rounding); // Stage 5 creates the four final butterfly pairs spanning the two groups. for (int i = 0; i < 4; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[i], step[i + 4], out output[i], out output[i + 4]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 4], out buffer0[i], out buffer0[i + 4]); } - // Stage 6 applies the remaining odd angles. Each result is placed in step for the fixed ADST permutation. - step[0] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[4], output[0], cospi[60], output[1], cosBit, in rounding); - step[1] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[60], output[0], -cospi[4], output[1], cosBit, in rounding); - step[2] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[20], output[2], cospi[44], output[3], cosBit, in rounding); - step[3] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[44], output[2], -cospi[20], output[3], cosBit, in rounding); - step[4] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[36], output[4], cospi[28], output[5], cosBit, in rounding); - step[5] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[28], output[4], -cospi[36], output[5], cosBit, in rounding); - step[6] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[52], output[6], cospi[12], output[7], cosBit, in rounding); - step[7] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[12], output[6], -cospi[52], output[7], cosBit, in rounding); - - // Stage 7 is the normative ADST output permutation. - output[0] = step[1]; - output[1] = step[6]; - output[2] = step[3]; - output[3] = step[4]; - output[4] = step[5]; - output[5] = step[2]; - output[6] = step[7]; - output[7] = step[0]; + // Stage 6 applies the remaining odd-angle rotations. + buffer1[0] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[4], buffer0[0], cospi[60], buffer0[1], cosBit, in rounding); + buffer1[1] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[60], buffer0[0], -cospi[4], buffer0[1], cosBit, in rounding); + buffer1[2] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[20], buffer0[2], cospi[44], buffer0[3], cosBit, in rounding); + buffer1[3] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[44], buffer0[2], -cospi[20], buffer0[3], cosBit, in rounding); + buffer1[4] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[36], buffer0[4], cospi[28], buffer0[5], cosBit, in rounding); + buffer1[5] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[28], buffer0[4], -cospi[36], buffer0[5], cosBit, in rounding); + buffer1[6] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[52], buffer0[6], cospi[12], buffer0[7], cosBit, in rounding); + buffer1[7] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[12], buffer0[6], -cospi[52], buffer0[7], cosBit, in rounding); + + ReadOnlySpan outputOrder = Adst8OutputOrder; + + // Stage 7 maps the rotated values to ascending AV1 ADST coefficient order. + for (int i = 0; i < 8; i++) + { + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); + } } /// /// Applies the sixteen-point forward asymmetric discrete sine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values and first transform-stage buffer. - /// The second transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Adst16( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - // Stage 1 applies the bit-reversed ADST input order and its alternating signs. - output[0] = input[0]; - output[1] = Av1ForwardTransformArithmetic.Negate(input[15]); - output[2] = Av1ForwardTransformArithmetic.Negate(input[7]); - output[3] = input[8]; - output[4] = Av1ForwardTransformArithmetic.Negate(input[3]); - output[5] = input[12]; - output[6] = input[4]; - output[7] = Av1ForwardTransformArithmetic.Negate(input[11]); - output[8] = Av1ForwardTransformArithmetic.Negate(input[1]); - output[9] = input[14]; - output[10] = input[6]; - output[11] = Av1ForwardTransformArithmetic.Negate(input[9]); - output[12] = input[2]; - output[13] = Av1ForwardTransformArithmetic.Negate(input[13]); - output[14] = Av1ForwardTransformArithmetic.Negate(input[5]); - output[15] = input[10]; - - // Stage 2 rotates the second pair in each group of four and copies the first pair unchanged. + // Stage 1 is the bit-reversed ADST input order with the normative alternating signs. + buffer0[0] = Load(ref values, inputStride, 0); + buffer0[1] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 15)); + buffer0[2] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 7)); + buffer0[3] = Load(ref values, inputStride, 8); + buffer0[4] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 3)); + buffer0[5] = Load(ref values, inputStride, 12); + buffer0[6] = Load(ref values, inputStride, 4); + buffer0[7] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 11)); + buffer0[8] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 1)); + buffer0[9] = Load(ref values, inputStride, 14); + buffer0[10] = Load(ref values, inputStride, 6); + buffer0[11] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 9)); + buffer0[12] = Load(ref values, inputStride, 2); + buffer0[13] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 13)); + buffer0[14] = Av1ForwardTransformArithmetic.Negate(Load(ref values, inputStride, 5)); + buffer0[15] = Load(ref values, inputStride, 10); + + // Stage 2 rotates the second pair in each group of four while copying the first pair unchanged. for (int group = 0; group < 16; group += 4) { - step[group] = output[group]; - step[group + 1] = output[group + 1]; - Av1ForwardTransformArithmetic.Butterfly( - cospi[32], - cospi[32], - output[group + 2], - output[group + 3], - out step[group + 2], - out step[group + 3], - cosBit, - in rounding); + buffer1[group] = buffer0[group]; + buffer1[group + 1] = buffer0[group + 1]; + Butterfly(cospi[32], cospi[32], buffer0[group + 2], buffer0[group + 3], ref buffer1, group + 2, group + 3, cosBit, in rounding); } // Stage 3 combines adjacent pairs within each group of four. @@ -218,10 +225,10 @@ internal static partial class Av1ForwardTransformOperations for (int i = 0; i < 2; i++) { Av1ForwardTransformArithmetic.AddSubtract( - step[group + i], - step[group + i + 2], - out output[group + i], - out output[group + i + 2]); + buffer1[group + i], + buffer1[group + i + 2], + out buffer0[group + i], + out buffer0[group + i + 2]); } } @@ -230,17 +237,20 @@ internal static partial class Av1ForwardTransformOperations { for (int i = 0; i < 4; i++) { - step[group + i] = output[group + i]; + buffer1[group + i] = buffer0[group + i]; } - step[group + 4] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[16], output[group + 4], cospi[48], output[group + 5], cosBit, in rounding); - step[group + 5] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[48], output[group + 4], -cospi[16], output[group + 5], cosBit, in rounding); - step[group + 6] = Av1ForwardTransformArithmetic.HalfButterfly( - -cospi[48], output[group + 6], cospi[16], output[group + 7], cosBit, in rounding); - step[group + 7] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[16], output[group + 6], cospi[48], output[group + 7], cosBit, in rounding); + buffer1[group + 4] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], buffer0[group + 4], cospi[48], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 5] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[48], buffer0[group + 4], -cospi[16], buffer0[group + 5], cosBit, in rounding); + + buffer1[group + 6] = Av1ForwardTransformArithmetic.HalfButterfly( + -cospi[48], buffer0[group + 6], cospi[16], buffer0[group + 7], cosBit, in rounding); + + buffer1[group + 7] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[16], buffer0[group + 6], cospi[48], buffer0[group + 7], cosBit, in rounding); } // Stage 5 combines the lower and upper quartets within each eight-value group. @@ -249,63 +259,56 @@ internal static partial class Av1ForwardTransformOperations for (int i = 0; i < 4; i++) { Av1ForwardTransformArithmetic.AddSubtract( - step[group + i], - step[group + i + 4], - out output[group + i], - out output[group + i + 4]); + buffer1[group + i], + buffer1[group + i + 4], + out buffer0[group + i], + out buffer0[group + i + 4]); } } // Stage 6 rotates the upper octet by pi/16 while retaining the completed lower octet. for (int i = 0; i < 8; i++) { - step[i] = output[i]; + buffer1[i] = buffer0[i]; } - step[8] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[8], output[8], cospi[56], output[9], cosBit, in rounding); - step[9] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[56], output[8], -cospi[8], output[9], cosBit, in rounding); - step[10] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[40], output[10], cospi[24], output[11], cosBit, in rounding); - step[11] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[24], output[10], -cospi[40], output[11], cosBit, in rounding); - step[12] = Av1ForwardTransformArithmetic.HalfButterfly( - -cospi[56], output[12], cospi[8], output[13], cosBit, in rounding); - step[13] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[8], output[12], cospi[56], output[13], cosBit, in rounding); - step[14] = Av1ForwardTransformArithmetic.HalfButterfly( - -cospi[24], output[14], cospi[40], output[15], cosBit, in rounding); - step[15] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[40], output[14], cospi[24], output[15], cosBit, in rounding); + buffer1[8] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[8], cospi[56], buffer0[9], cosBit, in rounding); + buffer1[9] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[56], buffer0[8], -cospi[8], buffer0[9], cosBit, in rounding); + buffer1[10] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[10], cospi[24], buffer0[11], cosBit, in rounding); + buffer1[11] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[24], buffer0[10], -cospi[40], buffer0[11], cosBit, in rounding); + buffer1[12] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[56], buffer0[12], cospi[8], buffer0[13], cosBit, in rounding); + buffer1[13] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[8], buffer0[12], cospi[56], buffer0[13], cosBit, in rounding); + buffer1[14] = Av1ForwardTransformArithmetic.HalfButterfly(-cospi[24], buffer0[14], cospi[40], buffer0[15], cosBit, in rounding); + buffer1[15] = Av1ForwardTransformArithmetic.HalfButterfly(cospi[40], buffer0[14], cospi[24], buffer0[15], cosBit, in rounding); // Stage 7 creates the eight final butterfly pairs spanning both octets. for (int i = 0; i < 8; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[i], step[i + 8], out output[i], out output[i + 8]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[i + 8], out buffer0[i], out buffer0[i + 8]); } - // Stage 8 applies the final odd-angle rotations before the fixed output permutation. - ReadOnlySpan firstWeights = [2, 10, 18, 26, 34, 42, 50, 58]; + ReadOnlySpan finalWeights = Adst16FinalWeights; + // Stage 8 applies the final odd-angle rotations. The compact weight table preserves their normative order + // without allocating a per-call array or duplicating the complementary cosine-index calculation. for (int pair = 0; pair < 8; pair++) { - int first = firstWeights[pair]; + int first = finalWeights[pair]; int second = 64 - first; int index = pair * 2; + buffer1[index] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[first], buffer0[index], cospi[second], buffer0[index + 1], cosBit, in rounding); - step[index] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[first], output[index], cospi[second], output[index + 1], cosBit, in rounding); - step[index + 1] = Av1ForwardTransformArithmetic.HalfButterfly( - cospi[second], output[index], -cospi[first], output[index + 1], cosBit, in rounding); + buffer1[index + 1] = Av1ForwardTransformArithmetic.HalfButterfly( + cospi[second], buffer0[index], -cospi[first], buffer0[index + 1], cosBit, in rounding); } - // Stage 9 maps the rotated input to ascending AV1 ADST coefficient order. - ReadOnlySpan permutation = [1, 14, 3, 12, 5, 10, 7, 8, 9, 6, 11, 4, 13, 2, 15, 0]; + ReadOnlySpan outputOrder = Adst16OutputOrder; + // Stage 9 maps the rotated values to ascending AV1 ADST coefficient order. for (int i = 0; i < 16; i++) { - output[i] = step[permutation[i]]; + Store(ref values, outputStride, i, buffer1[outputOrder[i]]); } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs index 9126069d0..44c091ec2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct16.cs @@ -12,109 +12,207 @@ internal static partial class Av1ForwardTransformOperations /// Applies the sixteen-point forward discrete cosine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The fixed transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Dct16( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - Av1ForwardTransformArithmetic.AddSubtract(input[0], input[15], out output[0], out output[15]); - Av1ForwardTransformArithmetic.AddSubtract(input[1], input[14], out output[1], out output[14]); - Av1ForwardTransformArithmetic.AddSubtract(input[2], input[13], out output[2], out output[13]); - Av1ForwardTransformArithmetic.AddSubtract(input[3], input[12], out output[3], out output[12]); - Av1ForwardTransformArithmetic.AddSubtract(input[4], input[11], out output[4], out output[11]); - Av1ForwardTransformArithmetic.AddSubtract(input[5], input[10], out output[5], out output[10]); - Av1ForwardTransformArithmetic.AddSubtract(input[6], input[9], out output[6], out output[9]); - Av1ForwardTransformArithmetic.AddSubtract(input[7], input[8], out output[7], out output[8]); - - // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - Av1ForwardTransformArithmetic.AddSubtract(output[0], output[7], out step[0], out step[7]); - Av1ForwardTransformArithmetic.AddSubtract(output[1], output[6], out step[1], out step[6]); - Av1ForwardTransformArithmetic.AddSubtract(output[2], output[5], out step[2], out step[5]); - Av1ForwardTransformArithmetic.AddSubtract(output[3], output[4], out step[3], out step[4]); - - step[8] = output[8]; - step[9] = output[9]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[10], output[13], out step[10], out step[13], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[11], output[12], out step[11], out step[12], cosBit, in rounding); - step[14] = output[14]; - step[15] = output[15]; - - // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. - Av1ForwardTransformArithmetic.AddSubtract(step[0], step[3], out output[0], out output[3]); - Av1ForwardTransformArithmetic.AddSubtract(step[1], step[2], out output[1], out output[2]); - - output[4] = step[4]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[5], step[6], out output[5], out output[6], cosBit, in rounding); - output[7] = step[7]; - Av1ForwardTransformArithmetic.AddSubtract(step[8], step[11], out output[8], out output[11]); - Av1ForwardTransformArithmetic.AddSubtract(step[9], step[10], out output[9], out output[10]); - Av1ForwardTransformArithmetic.AddSubtract(step[14], step[13], out output[14], out output[13]); - Av1ForwardTransformArithmetic.AddSubtract(step[15], step[12], out output[15], out output[12]); - - // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. - Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], output[0], output[1], out step[0], out step[1], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], output[3], output[2], out step[2], out step[3], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(output[4], output[5], out step[4], out step[5]); - Av1ForwardTransformArithmetic.AddSubtract(output[7], output[6], out step[7], out step[6]); - - step[8] = output[8]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[9], output[14], out step[9], out step[14], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[10], output[13], out step[10], out step[13], cosBit, in rounding); - step[11] = output[11]; - step[12] = output[12]; - step[15] = output[15]; - - // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], step[7], step[4], out output[4], out output[7], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], step[6], step[5], out output[5], out output[6], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(step[8], step[9], out output[8], out output[9]); - Av1ForwardTransformArithmetic.AddSubtract(step[11], step[10], out output[11], out output[10]); - Av1ForwardTransformArithmetic.AddSubtract(step[12], step[13], out output[12], out output[13]); - Av1ForwardTransformArithmetic.AddSubtract(step[15], step[14], out output[15], out output[14]); - - // Stage 6 applies the final pi/32 odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - Av1ForwardTransformArithmetic.Butterfly(cospi[4], cospi[60], output[15], output[8], out step[8], out step[15], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[36], cospi[28], output[14], output[9], out step[9], out step[14], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[20], cospi[44], output[13], output[10], out step[10], out step[13], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[52], cospi[12], output[12], output[11], out step[11], out step[12], cosBit, in rounding); - - // Stage 7 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[8]; - output[2] = step[4]; - output[3] = step[12]; - output[4] = step[2]; - output[5] = step[10]; - output[6] = step[6]; - output[7] = step[14]; - output[8] = step[1]; - output[9] = step[9]; - output[10] = step[5]; - output[11] = step[13]; - output[12] = step[3]; - output[13] = step[11]; - output[14] = step[7]; - output[15] = step[15]; + + // Stage 1 forms the mirror-symmetric pairs consumed by the recursive even and odd factorizations. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 15 - i), + out buffer0[i], + out buffer0[15 - i]); + } + + // Stage 2 begins the recursive factorization of the even half and rotates the central odd pairs by pi/4. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + // Stage 3 reduces both eight-value groups into the four-value units consumed by the terminal rotations. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + } + + // The even coefficients become final at stages 4 and 5, so they are written directly to their AV1 order. + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out TValue output0, + out TValue output8, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out TValue output4, + out TValue output12, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[8], + cospi[56], + buffer1[7], + buffer1[4], + out TValue output2, + out TValue output14, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[40], + cospi[24], + buffer1[6], + buffer1[5], + out TValue output10, + out TValue output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + + // Stage 6 applies the final pi/32 odd-frequency rotations. The following stores perform only the normative + // coefficient permutation, so each rotation result is named by its final destination. + Av1ForwardTransformArithmetic.Butterfly( + cospi[4], + cospi[60], + buffer0[15], + buffer0[8], + out TValue output1, + out TValue output15, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[36], + cospi[28], + buffer0[14], + buffer0[9], + out TValue output9, + out TValue output7, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[20], + cospi[44], + buffer0[13], + buffer0[10], + out TValue output5, + out TValue output11, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[52], + cospi[12], + buffer0[12], + buffer0[11], + out TValue output13, + out TValue output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); + Store(ref values, outputStride, 8, output8); + Store(ref values, outputStride, 9, output9); + Store(ref values, outputStride, 10, output10); + Store(ref values, outputStride, 11, output11); + Store(ref values, outputStride, 12, output12); + Store(ref values, outputStride, 13, output13); + Store(ref values, outputStride, 14, output14); + Store(ref values, outputStride, 15, output15); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs index 9d3d47d20..1a56d9372 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct32.cs @@ -12,213 +12,252 @@ internal static partial class Av1ForwardTransformOperations /// Applies the thirty-two-point forward discrete cosine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The fixed transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Dct32( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - for (int index = 0; index < 16; index++) + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 consumes the source block completely before any final coefficient is stored back into it. + for (int i = 0; i < 16; i++) { - Av1ForwardTransformArithmetic.AddSubtract(input[index], input[31 - index], out output[index], out output[31 - index]); + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 31 - i), + out buffer1[i], + out buffer1[31 - i]); } - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - for (int index = 0; index < 8; index++) + // Stage 2 starts the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 8; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); + } + + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[20 + i], + buffer1[27 - i], + out buffer0[20 + i], + out buffer0[27 - i], + cosBit, + in rounding); + } + + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); + } + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[11], + buffer0[12], + out buffer1[11], + out buffer1[12], + cosBit, + in rounding); + + for (int i = 0; i < 4; i++) { - Av1ForwardTransformArithmetic.AddSubtract(output[index], output[15 - index], out step[index], out step[15 - index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); } - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[20], output[27], out step[20], out step[27], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[21], output[26], out step[21], out step[26], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[22], output[25], out step[22], out step[25], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[23], output[24], out step[23], out step[24], cosBit, in rounding); - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - for (int index = 0; index < 4; index++) + // Stage 4 continues the factorization as independent eight-value groups. + for (int i = 0; i < 2; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[index], step[7 - index], out output[index], out output[7 - index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); } - output[8] = step[8]; - output[9] = step[9]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[10], step[13], out output[10], out output[13], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[11], step[12], out output[11], out output[12], cosBit, in rounding); - output[14] = step[14]; - output[15] = step[15]; - for (int index = 0; index < 4; index++) + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer1[5], + buffer1[6], + out buffer0[5], + out buffer0[6], + cosBit, + in rounding); + + for (int i = 0; i < 2; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[16 + index], step[23 - index], out output[16 + index], out output[23 - index]); - Av1ForwardTransformArithmetic.AddSubtract(step[31 - index], step[24 + index], out output[31 - index], out output[24 + index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); } - // Stage 4 continues the factorization as independent eight-sample groups. - Av1ForwardTransformArithmetic.AddSubtract(output[0], output[3], out step[0], out step[3]); - Av1ForwardTransformArithmetic.AddSubtract(output[1], output[2], out step[1], out step[2]); - - step[4] = output[4]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[5], output[6], out step[5], out step[6], cosBit, in rounding); - step[7] = output[7]; - Av1ForwardTransformArithmetic.AddSubtract(output[8], output[11], out step[8], out step[11]); - Av1ForwardTransformArithmetic.AddSubtract(output[9], output[10], out step[9], out step[10]); - Av1ForwardTransformArithmetic.AddSubtract(output[14], output[13], out step[14], out step[13]); - Av1ForwardTransformArithmetic.AddSubtract(output[15], output[12], out step[15], out step[12]); - - step[16] = output[16]; - step[17] = output[17]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[18], output[29], out step[18], out step[29], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[19], output[28], out step[19], out step[28], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[20], output[27], out step[20], out step[27], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[21], output[26], out step[21], out step[26], cosBit, in rounding); - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[30] = output[30]; - step[31] = output[31]; - - // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. - Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], step[0], step[1], out output[0], out output[1], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], step[3], step[2], out output[2], out output[3], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(step[4], step[5], out output[4], out output[5]); - Av1ForwardTransformArithmetic.AddSubtract(step[7], step[6], out output[7], out output[6]); - - output[8] = step[8]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], step[9], step[14], out output[9], out output[14], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], step[10], step[13], out output[10], out output[13], cosBit, in rounding); - output[11] = step[11]; - output[12] = step[12]; - output[15] = step[15]; - Av1ForwardTransformArithmetic.AddSubtract(step[16], step[19], out output[16], out output[19]); - Av1ForwardTransformArithmetic.AddSubtract(step[17], step[18], out output[17], out output[18]); - Av1ForwardTransformArithmetic.AddSubtract(step[23], step[20], out output[23], out output[20]); - Av1ForwardTransformArithmetic.AddSubtract(step[22], step[21], out output[22], out output[21]); - Av1ForwardTransformArithmetic.AddSubtract(step[24], step[27], out output[24], out output[27]); - Av1ForwardTransformArithmetic.AddSubtract(step[25], step[26], out output[25], out output[26]); - Av1ForwardTransformArithmetic.AddSubtract(step[31], step[28], out output[31], out output[28]); - Av1ForwardTransformArithmetic.AddSubtract(step[30], step[29], out output[30], out output[29]); - - // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], output[7], output[4], out step[4], out step[7], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], output[6], output[5], out step[5], out step[6], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(output[8], output[9], out step[8], out step[9]); - Av1ForwardTransformArithmetic.AddSubtract(output[11], output[10], out step[11], out step[10]); - Av1ForwardTransformArithmetic.AddSubtract(output[12], output[13], out step[12], out step[13]); - Av1ForwardTransformArithmetic.AddSubtract(output[15], output[14], out step[15], out step[14]); - - step[16] = output[16]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], output[17], output[30], out step[17], out step[30], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], output[18], output[29], out step[18], out step[29], cosBit, in rounding); - step[19] = output[19]; - step[20] = output[20]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], output[21], output[26], out step[21], out step[26], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], output[22], output[25], out step[22], out step[25], cosBit, in rounding); - step[23] = output[23]; - step[24] = output[24]; - step[27] = output[27]; - step[28] = output[28]; - step[31] = output[31]; + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer1[19], + buffer1[28], + out buffer0[19], + out buffer0[28], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer1[20], + buffer1[27], + out buffer0[20], + out buffer0[27], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. Final coefficients are + // retired directly to the block instead of being copied through a third workspace. + ButterflyStore(cospi[32], cospi[32], buffer0[0], buffer0[1], ref values, outputStride, 0, 16, cosBit, in rounding); + ButterflyStore(cospi[16], cospi[48], buffer0[3], buffer0[2], ref values, outputStride, 8, 24, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[16], + cospi[48], + buffer0[9], + buffer0[14], + out buffer1[9], + out buffer1[14], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[48], + -cospi[16], + buffer0[10], + buffer0[13], + out buffer1[10], + out buffer1[13], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[19], out buffer1[16], out buffer1[19]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[17], buffer0[18], out buffer1[17], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[20], out buffer1[23], out buffer1[20]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[22], buffer0[21], out buffer1[22], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[27], out buffer1[24], out buffer1[27]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[25], buffer0[26], out buffer1[25], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[28], out buffer1[31], out buffer1[28]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[30], buffer0[29], out buffer1[30], out buffer1[29]); + + // Stage 6 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + ButterflyStore(cospi[8], cospi[56], buffer1[7], buffer1[4], ref values, outputStride, 4, 28, cosBit, in rounding); + ButterflyStore(cospi[40], cospi[24], buffer1[6], buffer1[5], ref values, outputStride, 20, 12, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[8], + cospi[56], + buffer1[17], + buffer1[30], + out buffer0[17], + out buffer0[30], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[56], + -cospi[8], + buffer1[18], + buffer1[29], + out buffer0[18], + out buffer0[29], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[40], + cospi[24], + buffer1[21], + buffer1[26], + out buffer0[21], + out buffer0[26], + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + -cospi[24], + -cospi[40], + buffer1[22], + buffer1[25], + out buffer0[22], + out buffer0[25], + cosBit, + in rounding); // Stage 7 applies the pi/32 rotations to the next odd-frequency level. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - Av1ForwardTransformArithmetic.Butterfly(cospi[4], cospi[60], step[15], step[8], out output[8], out output[15], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[36], cospi[28], step[14], step[9], out output[9], out output[14], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[20], cospi[44], step[13], step[10], out output[10], out output[13], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[52], cospi[12], step[12], step[11], out output[11], out output[12], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(step[16], step[17], out output[16], out output[17]); - Av1ForwardTransformArithmetic.AddSubtract(step[19], step[18], out output[19], out output[18]); - Av1ForwardTransformArithmetic.AddSubtract(step[20], step[21], out output[20], out output[21]); - Av1ForwardTransformArithmetic.AddSubtract(step[23], step[22], out output[23], out output[22]); - Av1ForwardTransformArithmetic.AddSubtract(step[24], step[25], out output[24], out output[25]); - Av1ForwardTransformArithmetic.AddSubtract(step[27], step[26], out output[27], out output[26]); - Av1ForwardTransformArithmetic.AddSubtract(step[28], step[29], out output[28], out output[29]); - Av1ForwardTransformArithmetic.AddSubtract(step[31], step[30], out output[31], out output[30]); - - // Stage 8 merges the final odd-frequency pairs before their terminal rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - Av1ForwardTransformArithmetic.Butterfly(cospi[2], cospi[62], output[31], output[16], out step[16], out step[31], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[34], cospi[30], output[30], output[17], out step[17], out step[30], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[18], cospi[46], output[29], output[18], out step[18], out step[29], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[50], cospi[14], output[28], output[19], out step[19], out step[28], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[10], cospi[54], output[27], output[20], out step[20], out step[27], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[42], cospi[22], output[26], output[21], out step[21], out step[26], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[26], cospi[38], output[25], output[22], out step[22], out step[25], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[58], cospi[6], output[24], output[23], out step[23], out step[24], cosBit, in rounding); - - // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[16]; - output[2] = step[8]; - output[3] = step[24]; - output[4] = step[4]; - output[5] = step[20]; - output[6] = step[12]; - output[7] = step[28]; - output[8] = step[2]; - output[9] = step[18]; - output[10] = step[10]; - output[11] = step[26]; - output[12] = step[6]; - output[13] = step[22]; - output[14] = step[14]; - output[15] = step[30]; - output[16] = step[1]; - output[17] = step[17]; - output[18] = step[9]; - output[19] = step[25]; - output[20] = step[5]; - output[21] = step[21]; - output[22] = step[13]; - output[23] = step[29]; - output[24] = step[3]; - output[25] = step[19]; - output[26] = step[11]; - output[27] = step[27]; - output[28] = step[7]; - output[29] = step[23]; - output[30] = step[15]; - output[31] = step[31]; + ButterflyStore(cospi[4], cospi[60], buffer0[15], buffer0[8], ref values, outputStride, 2, 30, cosBit, in rounding); + ButterflyStore(cospi[36], cospi[28], buffer0[14], buffer0[9], ref values, outputStride, 18, 14, cosBit, in rounding); + ButterflyStore(cospi[20], cospi[44], buffer0[13], buffer0[10], ref values, outputStride, 10, 22, cosBit, in rounding); + ButterflyStore(cospi[52], cospi[12], buffer0[12], buffer0[11], ref values, outputStride, 26, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + // Stages 8 and 9 fuse the terminal pi/64 rotations with the output permutation because none of their results + // are consumed by another arithmetic stage. + ButterflyStore(cospi[2], cospi[62], buffer1[31], buffer1[16], ref values, outputStride, 1, 31, cosBit, in rounding); + ButterflyStore(cospi[34], cospi[30], buffer1[30], buffer1[17], ref values, outputStride, 17, 15, cosBit, in rounding); + ButterflyStore(cospi[18], cospi[46], buffer1[29], buffer1[18], ref values, outputStride, 9, 23, cosBit, in rounding); + ButterflyStore(cospi[50], cospi[14], buffer1[28], buffer1[19], ref values, outputStride, 25, 7, cosBit, in rounding); + ButterflyStore(cospi[10], cospi[54], buffer1[27], buffer1[20], ref values, outputStride, 5, 27, cosBit, in rounding); + ButterflyStore(cospi[42], cospi[22], buffer1[26], buffer1[21], ref values, outputStride, 21, 11, cosBit, in rounding); + ButterflyStore(cospi[26], cospi[38], buffer1[25], buffer1[22], ref values, outputStride, 13, 19, cosBit, in rounding); + ButterflyStore(cospi[58], cospi[6], buffer1[24], buffer1[23], ref values, outputStride, 29, 3, cosBit, in rounding); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs index 824fbd612..bd53e36ce 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct4.cs @@ -1,6 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// @@ -12,30 +14,159 @@ internal static partial class Av1ForwardTransformOperations /// Applies the four-point forward discrete cosine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The fixed transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Dct4( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { - // Libaom forms both outputs of each mirror pair together. This preserves the saturating Int16 AddSub - // primitive used by Highway while the Int32 and scalar specializations retain their native arithmetic. - Av1ForwardTransformArithmetic.AddSubtract(input[0], input[3], out output[0], out output[3]); - Av1ForwardTransformArithmetic.AddSubtract(input[1], input[2], out output[1], out output[2]); - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], output[0], output[1], out step[0], out step[2], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], output[3], output[2], out step[1], out step[3], cosBit, in rounding); - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; + TValue input0 = Load(ref values, inputStride, 0); + TValue input1 = Load(ref values, inputStride, 1); + TValue input2 = Load(ref values, inputStride, 2); + TValue input3 = Load(ref values, inputStride, 3); + + // The paired stage keeps the axes in their native lane representation. Packed short lanes therefore retain + // Highway's saturating add/subtract behavior before the widening butterfly multiplication. + Av1ForwardTransformArithmetic.AddSubtract(input0, input3, out buffer0[0], out buffer0[3]); + Av1ForwardTransformArithmetic.AddSubtract(input1, input2, out buffer0[1], out buffer0[2]); + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer0[0], + buffer0[1], + out TValue output0, + out TValue output2, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer0[3], + buffer0[2], + out TValue output1, + out TValue output3, + cosBit, + in rounding); + + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + } + + /// + /// Loads one scalar or SIMD transform value from strided block storage. + /// + /// The scalar or SIMD value to load. + /// The first value in the transform block. + /// The byte distance between consecutive transform positions. + /// The transform position to load. + /// The requested transform value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TValue Load(ref byte values, nint stride, int index) + where TValue : struct + => Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref values, index * stride)); + + /// + /// Stores one scalar or SIMD transform value in strided block storage. + /// + /// The scalar or SIMD value to store. + /// The first value in the transform block. + /// The byte distance between consecutive transform positions. + /// The transform position to store. + /// The transform value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Store(ref byte values, nint stride, int index, TValue value) + where TValue : struct + => Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref values, index * stride), value); + + /// + /// Applies one paired rotation and stores both results directly in the strided transform block. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The first fixed-point rotation weight. + /// The second fixed-point rotation weight. + /// The first rotation input. + /// The second rotation input. + /// The first value in the transform block. + /// The byte distance between consecutive transform positions. + /// The transform position for the first result. + /// The transform position for the second result. + /// The fixed-point precision of the rotation weights. + /// The lane-width-specific rounding constants. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ButterflyStore( + int weight0, + int weight1, + TValue input0, + TValue input1, + ref byte values, + nint stride, + int outputIndex0, + int outputIndex1, + int cosBit, + in Av1TransformRounding rounding) + where TValue : struct + { + Av1ForwardTransformArithmetic.Butterfly( + weight0, + weight1, + input0, + input1, + out TValue output0, + out TValue output1, + cosBit, + in rounding); + + Store(ref values, stride, outputIndex0, output0); + Store(ref values, stride, outputIndex1, output1); } + + /// + /// Applies one paired rotation into two positions of a transform-stage buffer. + /// + /// The scalar or SIMD value containing the independent transform axes. + /// The first fixed-point rotation weight. + /// The second fixed-point rotation weight. + /// The first rotation input. + /// The second rotation input. + /// The transform-stage buffer receiving both results. + /// The buffer position for the first result. + /// The buffer position for the second result. + /// The fixed-point precision of the rotation weights. + /// The lane-width-specific rounding constants. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Butterfly( + int weight0, + int weight1, + TValue input0, + TValue input1, + ref Av1TransformVector output, + int outputIndex0, + int outputIndex1, + int cosBit, + in Av1TransformRounding rounding) + where TValue : struct + => Av1ForwardTransformArithmetic.Butterfly( + weight0, + weight1, + input0, + input1, + out output[outputIndex0], + out output[outputIndex1], + cosBit, + in rounding); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs index df9175d5e..88585396c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct64.cs @@ -8,430 +8,278 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; /// internal static partial class Av1ForwardTransformOperations { + /// + /// Identifies coefficient positions whose final value resides in the first stage buffer. + /// + private const ulong Dct64Buffer0OutputMask = + (1UL << 2) | (1UL << 6) | (1UL << 8) | (1UL << 10) | (1UL << 14) | + (1UL << 18) | (1UL << 22) | (1UL << 24) | (1UL << 26) | (1UL << 30) | + (1UL << 34) | (1UL << 38) | (1UL << 40) | (1UL << 42) | (1UL << 46) | + (1UL << 50) | (1UL << 54) | (1UL << 56) | (1UL << 58) | (1UL << 62); + + /// + /// Gets the stage-nine rotation order for the middle quarter of the sixty-four-point DCT. + /// + private static ReadOnlySpan Dct64Stage9RotationOrder => [2, 34, 18, 50, 10, 42, 26, 58]; + + /// + /// Gets the stage-ten rotation order for the upper half of the sixty-four-point DCT. + /// + private static ReadOnlySpan Dct64Stage10RotationOrder => [1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61]; + + /// + /// Gets the mapping from coefficient order to the final staged value. + /// + private static ReadOnlySpan Dct64OutputOrder => + [ + 0, 32, 16, 48, 8, 40, 24, 56, 4, 36, 20, 52, 12, 44, 28, 60, + 2, 34, 18, 50, 10, 42, 26, 58, 6, 38, 22, 54, 14, 46, 30, 62, + 1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61, + 3, 35, 19, 51, 11, 43, 27, 59, 7, 39, 23, 55, 15, 47, 31, 63, + ]; + /// /// Applies the sixty-four-point forward discrete cosine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The fixed transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Dct64( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - for (int index = 0; index < 32; index++) + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); + + // Stage 1 consumes every spatial value before the strided block becomes available for final coefficients. + for (int i = 0; i < 32; i++) { - Av1ForwardTransformArithmetic.AddSubtract(input[index], input[63 - index], out output[index], out output[63 - index]); + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, i), + Load(ref values, inputStride, 63 - i), + out buffer0[i], + out buffer0[63 - i]); } - // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. - ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); - Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - for (int index = 0; index < 16; index++) + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd-frequency pairs. + for (int i = 0; i < 16; i++) { - Av1ForwardTransformArithmetic.AddSubtract(output[index], output[31 - index], out step[index], out step[31 - index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[31 - i], out buffer1[i], out buffer1[31 - i]); } - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - step[36] = output[36]; - step[37] = output[37]; - step[38] = output[38]; - step[39] = output[39]; - for (int index = 0; index < 8; index++) + for (int i = 0; i < 8; i++) { - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], cospi[32], output[40 + index], output[55 - index], out step[40 + index], out step[55 - index], cosBit, in rounding); + Butterfly(-cospi[32], cospi[32], buffer0[40 + i], buffer0[55 - i], ref buffer1, 40 + i, 55 - i, cosBit, in rounding); } - step[56] = output[56]; - step[57] = output[57]; - step[58] = output[58]; - step[59] = output[59]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. - for (int index = 0; index < 8; index++) + // Stage 3 reduces the even half and folds the next odd-frequency groups into paired sums and differences. + for (int i = 0; i < 8; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[index], step[15 - index], out output[index], out output[15 - index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[15 - i], out buffer0[i], out buffer0[15 - i]); } - output[16] = step[16]; - output[17] = step[17]; - output[18] = step[18]; - output[19] = step[19]; - for (int index = 0; index < 4; index++) + for (int i = 0; i < 4; i++) { - Av1ForwardTransformArithmetic.Butterfly( - -cospi[32], cospi[32], step[20 + index], step[27 - index], out output[20 + index], out output[27 - index], cosBit, in rounding); + Butterfly(-cospi[32], cospi[32], buffer1[20 + i], buffer1[27 - i], ref buffer0, 20 + i, 27 - i, cosBit, in rounding); } - output[28] = step[28]; - output[29] = step[29]; - output[30] = step[30]; - output[31] = step[31]; - for (int index = 0; index < 8; index++) + for (int i = 0; i < 8; i++) { - Av1ForwardTransformArithmetic.AddSubtract( - step[32 + index], step[47 - index], out output[32 + index], out output[47 - index]); - Av1ForwardTransformArithmetic.AddSubtract( - step[63 - index], step[48 + index], out output[63 - index], out output[48 + index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[47 - i], out buffer0[32 + i], out buffer0[47 - i]); } - // Stage 4 continues the factorization as independent sixteen-sample groups. - for (int index = 0; index < 4; index++) + for (int i = 0; i < 8; i++) { - Av1ForwardTransformArithmetic.AddSubtract(output[index], output[7 - index], out step[index], out step[7 - index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[48 + i], out buffer0[63 - i], out buffer0[48 + i]); } - step[8] = output[8]; - step[9] = output[9]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[10], output[13], out step[10], out step[13], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[11], output[12], out step[11], out step[12], cosBit, in rounding); - step[14] = output[14]; - step[15] = output[15]; - for (int index = 0; index < 4; index++) + // Stage 4 continues the factorization as independent sixteen-value groups. + for (int i = 0; i < 4; i++) { - Av1ForwardTransformArithmetic.AddSubtract( - output[16 + index], output[23 - index], out step[16 + index], out step[23 - index]); - Av1ForwardTransformArithmetic.AddSubtract( - output[31 - index], output[24 + index], out step[31 - index], out step[24 + index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[i], buffer0[7 - i], out buffer1[i], out buffer1[7 - i]); } - step[32] = output[32]; - step[33] = output[33]; - step[34] = output[34]; - step[35] = output[35]; - for (int index = 0; index < 4; index++) + for (int i = 0; i < 2; i++) { - Av1ForwardTransformArithmetic.Butterfly( - -cospi[16], cospi[48], output[36 + index], output[59 - index], out step[36 + index], out step[59 - index], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly( - -cospi[48], -cospi[16], output[40 + index], output[55 - index], out step[40 + index], out step[55 - index], cosBit, in rounding); + Butterfly(-cospi[32], cospi[32], buffer0[10 + i], buffer0[13 - i], ref buffer1, 10 + i, 13 - i, cosBit, in rounding); } - step[44] = output[44]; - step[45] = output[45]; - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[50] = output[50]; - step[51] = output[51]; - step[60] = output[60]; - step[61] = output[61]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. - Av1ForwardTransformArithmetic.AddSubtract(step[0], step[3], out output[0], out output[3]); - Av1ForwardTransformArithmetic.AddSubtract(step[1], step[2], out output[1], out output[2]); - - output[4] = step[4]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], step[5], step[6], out output[5], out output[6], cosBit, in rounding); - output[7] = step[7]; - Av1ForwardTransformArithmetic.AddSubtract(step[8], step[11], out output[8], out output[11]); - Av1ForwardTransformArithmetic.AddSubtract(step[9], step[10], out output[9], out output[10]); - Av1ForwardTransformArithmetic.AddSubtract(step[14], step[13], out output[14], out output[13]); - Av1ForwardTransformArithmetic.AddSubtract(step[15], step[12], out output[15], out output[12]); - output[16] = step[16]; - output[17] = step[17]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], step[18], step[29], out output[18], out output[29], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], step[19], step[28], out output[19], out output[28], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], step[20], step[27], out output[20], out output[27], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], step[21], step[26], out output[21], out output[26], cosBit, in rounding); - output[22] = step[22]; - output[23] = step[23]; - output[24] = step[24]; - output[25] = step[25]; - output[30] = step[30]; - output[31] = step[31]; - for (int index = 0; index < 4; index++) + for (int i = 0; i < 4; i++) { - Av1ForwardTransformArithmetic.AddSubtract( - step[32 + index], step[39 - index], out output[32 + index], out output[39 - index]); - Av1ForwardTransformArithmetic.AddSubtract( - step[47 - index], step[40 + index], out output[47 - index], out output[40 + index]); - Av1ForwardTransformArithmetic.AddSubtract( - step[48 + index], step[55 - index], out output[48 + index], out output[55 - index]); - Av1ForwardTransformArithmetic.AddSubtract( - step[63 - index], step[56 + index], out output[63 - index], out output[56 + index]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[23 - i], out buffer1[16 + i], out buffer1[23 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[24 + i], out buffer1[31 - i], out buffer1[24 + i]); + Butterfly(-cospi[16], cospi[48], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); } - // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. - Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], output[0], output[1], out step[0], out step[1], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], output[3], output[2], out step[2], out step[3], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(output[4], output[5], out step[4], out step[5]); - Av1ForwardTransformArithmetic.AddSubtract(output[7], output[6], out step[7], out step[6]); - step[8] = output[8]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[16], cospi[48], output[9], output[14], out step[9], out step[14], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[48], -cospi[16], output[10], output[13], out step[10], out step[13], cosBit, in rounding); - step[11] = output[11]; - step[12] = output[12]; - step[15] = output[15]; - Av1ForwardTransformArithmetic.AddSubtract(output[16], output[19], out step[16], out step[19]); - Av1ForwardTransformArithmetic.AddSubtract(output[17], output[18], out step[17], out step[18]); - Av1ForwardTransformArithmetic.AddSubtract(output[23], output[20], out step[23], out step[20]); - Av1ForwardTransformArithmetic.AddSubtract(output[22], output[21], out step[22], out step[21]); - Av1ForwardTransformArithmetic.AddSubtract(output[24], output[27], out step[24], out step[27]); - Av1ForwardTransformArithmetic.AddSubtract(output[25], output[26], out step[25], out step[26]); - Av1ForwardTransformArithmetic.AddSubtract(output[31], output[28], out step[31], out step[28]); - Av1ForwardTransformArithmetic.AddSubtract(output[30], output[29], out step[30], out step[29]); - step[32] = output[32]; - step[33] = output[33]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], output[34], output[61], out step[34], out step[61], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], output[35], output[60], out step[35], out step[60], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], output[36], output[59], out step[36], out step[59], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], output[37], output[58], out step[37], out step[58], cosBit, in rounding); - step[38] = output[38]; - step[39] = output[39]; - step[40] = output[40]; - step[41] = output[41]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], output[42], output[53], out step[42], out step[53], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], output[43], output[52], out step[43], out step[52], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], output[44], output[51], out step[44], out step[51], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], output[45], output[50], out step[45], out step[50], cosBit, in rounding); - step[46] = output[46]; - step[47] = output[47]; - step[48] = output[48]; - step[49] = output[49]; - step[54] = output[54]; - step[55] = output[55]; - step[56] = output[56]; - step[57] = output[57]; - step[62] = output[62]; - step[63] = output[63]; - - // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], step[7], step[4], out output[4], out output[7], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], step[6], step[5], out output[5], out output[6], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(step[8], step[9], out output[8], out output[9]); - Av1ForwardTransformArithmetic.AddSubtract(step[11], step[10], out output[11], out output[10]); - Av1ForwardTransformArithmetic.AddSubtract(step[12], step[13], out output[12], out output[13]); - Av1ForwardTransformArithmetic.AddSubtract(step[15], step[14], out output[15], out output[14]); - output[16] = step[16]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[8], cospi[56], step[17], step[30], out output[17], out output[30], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[56], -cospi[8], step[18], step[29], out output[18], out output[29], cosBit, in rounding); - output[19] = step[19]; - output[20] = step[20]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[40], cospi[24], step[21], step[26], out output[21], out output[26], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[24], -cospi[40], step[22], step[25], out output[22], out output[25], cosBit, in rounding); - output[23] = step[23]; - output[24] = step[24]; - output[27] = step[27]; - output[28] = step[28]; - output[31] = step[31]; - for (int offset = 32; offset < 64; offset += 8) + for (int i = 4; i < 8; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[offset], step[offset + 3], out output[offset], out output[offset + 3]); - Av1ForwardTransformArithmetic.AddSubtract(step[offset + 1], step[offset + 2], out output[offset + 1], out output[offset + 2]); - Av1ForwardTransformArithmetic.AddSubtract(step[offset + 7], step[offset + 4], out output[offset + 7], out output[offset + 4]); - Av1ForwardTransformArithmetic.AddSubtract(step[offset + 6], step[offset + 5], out output[offset + 6], out output[offset + 5]); + Butterfly(-cospi[48], -cospi[16], buffer0[36 + i], buffer0[59 - i], ref buffer1, 36 + i, 59 - i, cosBit, in rounding); } - // Stage 8 applies the next level of odd-frequency rotations. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - Av1ForwardTransformArithmetic.Butterfly(cospi[4], cospi[60], output[15], output[8], out step[8], out step[15], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[36], cospi[28], output[14], output[9], out step[9], out step[14], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[20], cospi[44], output[13], output[10], out step[10], out step[13], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[52], cospi[12], output[12], output[11], out step[11], out step[12], cosBit, in rounding); - for (int offset = 16; offset < 32; offset += 4) + // Stage 5 reduces the sixteen-value groups into the eight-value DCT and ADST building blocks. + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[i], buffer1[3 - i], out buffer0[i], out buffer0[3 - i]); + } + + Butterfly(-cospi[32], cospi[32], buffer1[5], buffer1[6], ref buffer0, 5, 6, cosBit, in rounding); + + for (int i = 0; i < 2; i++) { - Av1ForwardTransformArithmetic.AddSubtract(output[offset], output[offset + 1], out step[offset], out step[offset + 1]); - Av1ForwardTransformArithmetic.AddSubtract(output[offset + 3], output[offset + 2], out step[offset + 3], out step[offset + 2]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8 + i], buffer1[11 - i], out buffer0[8 + i], out buffer0[11 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15 - i], buffer1[12 + i], out buffer0[15 - i], out buffer0[12 + i]); + Butterfly(-cospi[16], cospi[48], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); } - step[32] = output[32]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[4], cospi[60], output[33], output[62], out step[33], out step[62], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[60], -cospi[4], output[34], output[61], out step[34], out step[61], cosBit, in rounding); - step[35] = output[35]; - step[36] = output[36]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[36], cospi[28], output[37], output[58], out step[37], out step[58], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[28], -cospi[36], output[38], output[57], out step[38], out step[57], cosBit, in rounding); - step[39] = output[39]; - step[40] = output[40]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[20], cospi[44], output[41], output[54], out step[41], out step[54], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[44], -cospi[20], output[42], output[53], out step[42], out step[53], cosBit, in rounding); - step[43] = output[43]; - step[44] = output[44]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[52], cospi[12], output[45], output[50], out step[45], out step[50], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(-cospi[12], -cospi[52], output[46], output[49], out step[46], out step[49], cosBit, in rounding); - step[47] = output[47]; - step[48] = output[48]; - step[51] = output[51]; - step[52] = output[52]; - step[55] = output[55]; - step[56] = output[56]; - step[59] = output[59]; - step[60] = output[60]; - step[63] = output[63]; - - // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. - output[0] = step[0]; - output[1] = step[1]; - output[2] = step[2]; - output[3] = step[3]; - output[4] = step[4]; - output[5] = step[5]; - output[6] = step[6]; - output[7] = step[7]; - output[8] = step[8]; - output[9] = step[9]; - output[10] = step[10]; - output[11] = step[11]; - output[12] = step[12]; - output[13] = step[13]; - output[14] = step[14]; - output[15] = step[15]; - Av1ForwardTransformArithmetic.Butterfly(cospi[2], cospi[62], step[31], step[16], out output[16], out output[31], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[34], cospi[30], step[30], step[17], out output[17], out output[30], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[18], cospi[46], step[29], step[18], out output[18], out output[29], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[50], cospi[14], step[28], step[19], out output[19], out output[28], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[10], cospi[54], step[27], step[20], out output[20], out output[27], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[42], cospi[22], step[26], step[21], out output[21], out output[26], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[26], cospi[38], step[25], step[22], out output[22], out output[25], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[58], cospi[6], step[24], step[23], out output[23], out output[24], cosBit, in rounding); - for (int offset = 32; offset < 64; offset += 4) + for (int i = 2; i < 4; i++) { - Av1ForwardTransformArithmetic.AddSubtract(step[offset], step[offset + 1], out output[offset], out output[offset + 1]); - Av1ForwardTransformArithmetic.AddSubtract(step[offset + 3], step[offset + 2], out output[offset + 3], out output[offset + 2]); + Butterfly(-cospi[48], -cospi[16], buffer1[18 + i], buffer1[29 - i], ref buffer0, 18 + i, 29 - i, cosBit, in rounding); } + for (int i = 0; i < 4; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32 + i], buffer1[39 - i], out buffer0[32 + i], out buffer0[39 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[47 - i], buffer1[40 + i], out buffer0[47 - i], out buffer0[40 + i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[48 + i], buffer1[55 - i], out buffer0[48 + i], out buffer0[55 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63 - i], buffer1[56 + i], out buffer0[63 - i], out buffer0[56 + i]); + } + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd-frequency groups. + Butterfly(cospi[32], cospi[32], buffer0[0], buffer0[1], ref buffer1, 0, 1, cosBit, in rounding); + Butterfly(cospi[16], cospi[48], buffer0[3], buffer0[2], ref buffer1, 2, 3, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer0[5], out buffer1[4], out buffer1[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer0[6], out buffer1[7], out buffer1[6]); + Butterfly(-cospi[16], cospi[48], buffer0[9], buffer0[14], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(-cospi[48], -cospi[16], buffer0[10], buffer0[13], ref buffer1, 10, 13, cosBit, in rounding); + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16 + i], buffer0[19 - i], out buffer1[16 + i], out buffer1[19 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23 - i], buffer0[20 + i], out buffer1[23 - i], out buffer1[20 + i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24 + i], buffer0[27 - i], out buffer1[24 + i], out buffer1[27 - i]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31 - i], buffer0[28 + i], out buffer1[31 - i], out buffer1[28 + i]); + Butterfly(-cospi[8], cospi[56], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + for (int i = 2; i < 4; i++) + { + Butterfly(-cospi[56], -cospi[8], buffer0[34 + i], buffer0[61 - i], ref buffer1, 34 + i, 61 - i, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer0[42 + i], buffer0[53 - i], ref buffer1, 42 + i, 53 - i, cosBit, in rounding); + } + + // Stage 7 merges adjacent odd-frequency terms with the sign pattern required by the next rotations. + Butterfly(cospi[8], cospi[56], buffer1[7], buffer1[4], ref buffer0, 4, 7, cosBit, in rounding); + Butterfly(cospi[40], cospi[24], buffer1[6], buffer1[5], ref buffer0, 5, 6, cosBit, in rounding); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[8], buffer1[9], out buffer0[8], out buffer0[9]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[11], buffer1[10], out buffer0[11], out buffer0[10]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[12], buffer1[13], out buffer0[12], out buffer0[13]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[15], buffer1[14], out buffer0[15], out buffer0[14]); + Butterfly(-cospi[8], cospi[56], buffer1[17], buffer1[30], ref buffer0, 17, 30, cosBit, in rounding); + Butterfly(-cospi[56], -cospi[8], buffer1[18], buffer1[29], ref buffer0, 18, 29, cosBit, in rounding); + Butterfly(-cospi[40], cospi[24], buffer1[21], buffer1[26], ref buffer0, 21, 26, cosBit, in rounding); + Butterfly(-cospi[24], -cospi[40], buffer1[22], buffer1[25], ref buffer0, 22, 25, cosBit, in rounding); + + for (int group = 0; group < 4; group++) + { + int offset = group * 8; + + for (int i = 0; i < 2; i++) + { + Av1ForwardTransformArithmetic.AddSubtract( + buffer0[32 + offset + i], + buffer1[35 + offset - i], + out buffer0[32 + offset + i], + out buffer0[35 + offset - i]); + + Av1ForwardTransformArithmetic.AddSubtract( + buffer0[39 + offset - i], + buffer1[36 + offset + i], + out buffer0[39 + offset - i], + out buffer0[36 + offset + i]); + } + } + + // Stage 8 applies the next level of odd-frequency rotations. + Butterfly(cospi[4], cospi[60], buffer0[15], buffer0[8], ref buffer1, 8, 15, cosBit, in rounding); + Butterfly(cospi[36], cospi[28], buffer0[14], buffer0[9], ref buffer1, 9, 14, cosBit, in rounding); + Butterfly(cospi[20], cospi[44], buffer0[13], buffer0[10], ref buffer1, 10, 13, cosBit, in rounding); + Butterfly(cospi[52], cospi[12], buffer0[12], buffer0[11], ref buffer1, 11, 12, cosBit, in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[16], buffer0[17], out buffer1[16], out buffer1[17]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[19], buffer0[18], out buffer1[19], out buffer1[18]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[20], buffer0[21], out buffer1[20], out buffer1[21]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[23], buffer0[22], out buffer1[23], out buffer1[22]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[24], buffer0[25], out buffer1[24], out buffer1[25]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[27], buffer0[26], out buffer1[27], out buffer1[26]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[28], buffer0[29], out buffer1[28], out buffer1[29]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[31], buffer0[30], out buffer1[31], out buffer1[30]); + + Butterfly(-cospi[4], cospi[60], buffer0[33], buffer0[62], ref buffer1, 33, 62, cosBit, in rounding); + Butterfly(-cospi[60], -cospi[4], buffer0[34], buffer0[61], ref buffer1, 34, 61, cosBit, in rounding); + Butterfly(-cospi[36], cospi[28], buffer0[37], buffer0[58], ref buffer1, 37, 58, cosBit, in rounding); + Butterfly(-cospi[28], -cospi[36], buffer0[38], buffer0[57], ref buffer1, 38, 57, cosBit, in rounding); + Butterfly(-cospi[20], cospi[44], buffer0[41], buffer0[54], ref buffer1, 41, 54, cosBit, in rounding); + Butterfly(-cospi[44], -cospi[20], buffer0[42], buffer0[53], ref buffer1, 42, 53, cosBit, in rounding); + Butterfly(-cospi[52], cospi[12], buffer0[45], buffer0[50], ref buffer1, 45, 50, cosBit, in rounding); + Butterfly(-cospi[12], -cospi[52], buffer0[46], buffer0[49], ref buffer1, 46, 49, cosBit, in rounding); + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. The table preserves the + // non-linear rotation order while keeping the constants in compile-time data. + for (int i = 0; i < 8; i++) + { + int low = 16 + i; + int high = 31 - i; + int odd = Dct64Stage9RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer1[high], buffer1[low], ref buffer0, low, high, cosBit, in rounding); + } + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[32], buffer1[33], out buffer0[32], out buffer0[33]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[35], buffer1[34], out buffer0[35], out buffer0[34]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[36], buffer1[37], out buffer0[36], out buffer0[37]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[39], buffer1[38], out buffer0[39], out buffer0[38]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[40], buffer1[41], out buffer0[40], out buffer0[41]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[43], buffer1[42], out buffer0[43], out buffer0[42]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[44], buffer1[45], out buffer0[44], out buffer0[45]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[47], buffer1[46], out buffer0[47], out buffer0[46]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[48], buffer1[49], out buffer0[48], out buffer0[49]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[51], buffer1[50], out buffer0[51], out buffer0[50]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[52], buffer1[53], out buffer0[52], out buffer0[53]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[55], buffer1[54], out buffer0[55], out buffer0[54]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[56], buffer1[57], out buffer0[56], out buffer0[57]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[59], buffer1[58], out buffer0[59], out buffer0[58]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[60], buffer1[61], out buffer0[60], out buffer0[61]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[63], buffer1[62], out buffer0[63], out buffer0[62]); + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - step[4] = output[4]; - step[5] = output[5]; - step[6] = output[6]; - step[7] = output[7]; - step[8] = output[8]; - step[9] = output[9]; - step[10] = output[10]; - step[11] = output[11]; - step[12] = output[12]; - step[13] = output[13]; - step[14] = output[14]; - step[15] = output[15]; - step[16] = output[16]; - step[17] = output[17]; - step[18] = output[18]; - step[19] = output[19]; - step[20] = output[20]; - step[21] = output[21]; - step[22] = output[22]; - step[23] = output[23]; - step[24] = output[24]; - step[25] = output[25]; - step[26] = output[26]; - step[27] = output[27]; - step[28] = output[28]; - step[29] = output[29]; - step[30] = output[30]; - step[31] = output[31]; - Av1ForwardTransformArithmetic.Butterfly(cospi[1], cospi[63], output[63], output[32], out step[32], out step[63], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[33], cospi[31], output[62], output[33], out step[33], out step[62], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[17], cospi[47], output[61], output[34], out step[34], out step[61], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[49], cospi[15], output[60], output[35], out step[35], out step[60], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[9], cospi[55], output[59], output[36], out step[36], out step[59], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[41], cospi[23], output[58], output[37], out step[37], out step[58], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[25], cospi[39], output[57], output[38], out step[38], out step[57], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[57], cospi[7], output[56], output[39], out step[39], out step[56], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[5], cospi[59], output[55], output[40], out step[40], out step[55], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[37], cospi[27], output[54], output[41], out step[41], out step[54], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[21], cospi[43], output[53], output[42], out step[42], out step[53], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[53], cospi[11], output[52], output[43], out step[43], out step[52], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[13], cospi[51], output[51], output[44], out step[44], out step[51], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[45], cospi[19], output[50], output[45], out step[45], out step[50], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[29], cospi[35], output[49], output[46], out step[46], out step[49], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[61], cospi[3], output[48], output[47], out step[47], out step[48], cosBit, in rounding); - - // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. - output[0] = step[0]; - output[1] = step[32]; - output[2] = step[16]; - output[3] = step[48]; - output[4] = step[8]; - output[5] = step[40]; - output[6] = step[24]; - output[7] = step[56]; - output[8] = step[4]; - output[9] = step[36]; - output[10] = step[20]; - output[11] = step[52]; - output[12] = step[12]; - output[13] = step[44]; - output[14] = step[28]; - output[15] = step[60]; - output[16] = step[2]; - output[17] = step[34]; - output[18] = step[18]; - output[19] = step[50]; - output[20] = step[10]; - output[21] = step[42]; - output[22] = step[26]; - output[23] = step[58]; - output[24] = step[6]; - output[25] = step[38]; - output[26] = step[22]; - output[27] = step[54]; - output[28] = step[14]; - output[29] = step[46]; - output[30] = step[30]; - output[31] = step[62]; - output[32] = step[1]; - output[33] = step[33]; - output[34] = step[17]; - output[35] = step[49]; - output[36] = step[9]; - output[37] = step[41]; - output[38] = step[25]; - output[39] = step[57]; - output[40] = step[5]; - output[41] = step[37]; - output[42] = step[21]; - output[43] = step[53]; - output[44] = step[13]; - output[45] = step[45]; - output[46] = step[29]; - output[47] = step[61]; - output[48] = step[3]; - output[49] = step[35]; - output[50] = step[19]; - output[51] = step[51]; - output[52] = step[11]; - output[53] = step[43]; - output[54] = step[27]; - output[55] = step[59]; - output[56] = step[7]; - output[57] = step[39]; - output[58] = step[23]; - output[59] = step[55]; - output[60] = step[15]; - output[61] = step[47]; - output[62] = step[31]; - output[63] = step[63]; + for (int i = 0; i < 16; i++) + { + int low = 32 + i; + int high = 63 - i; + int odd = Dct64Stage10RotationOrder[i]; + Butterfly(cospi[odd], cospi[64 - odd], buffer0[high], buffer0[low], ref buffer1, low, high, cosBit, in rounding); + } + + // Stage 11 applies the terminal permutation. The fused stages omit pass-through copies, so sources 4-7 and + // 16-31 remain in buffer0 at retirement, + // while every other source resides in buffer1. The mask maps that ownership through AV1 coefficient order. + ReadOnlySpan outputOrder = Dct64OutputOrder; + + for (int i = 0; i < 64; i++) + { + int sourceIndex = outputOrder[i]; + TValue value = ((Dct64Buffer0OutputMask >> i) & 1) != 0 ? buffer0[sourceIndex] : buffer1[sourceIndex]; + + Store(ref values, outputStride, i, value); + } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs index dee0fa89b..4b9ad0951 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Dct8.cs @@ -12,55 +12,114 @@ internal static partial class Av1ForwardTransformOperations /// Applies the eight-point forward discrete cosine transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The fixed transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static void Dct8( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct { - // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. - Av1ForwardTransformArithmetic.AddSubtract(input[0], input[7], out output[0], out output[7]); - Av1ForwardTransformArithmetic.AddSubtract(input[1], input[6], out output[1], out output[6]); - Av1ForwardTransformArithmetic.AddSubtract(input[2], input[5], out output[2], out output[5]); - Av1ForwardTransformArithmetic.AddSubtract(input[3], input[4], out output[3], out output[4]); - - // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); Av1TransformRounding rounding = Av1ForwardTransformArithmetic.CreateRounding(cosBit); - Av1ForwardTransformArithmetic.AddSubtract(output[0], output[3], out step[0], out step[3]); - Av1ForwardTransformArithmetic.AddSubtract(output[1], output[2], out step[1], out step[2]); - step[4] = output[4]; - Av1ForwardTransformArithmetic.Butterfly(-cospi[32], cospi[32], output[5], output[6], out step[5], out step[6], cosBit, in rounding); - step[7] = output[7]; + // Stages 1 and 2 split the even and odd terms. The asymmetric destinations mirror Highway's buffer + // ownership, allowing the later even butterflies to write their final coefficients directly to the block. + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 0), + Load(ref values, inputStride, 7), + out buffer0[0], + out buffer1[7]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 1), + Load(ref values, inputStride, 6), + out buffer0[1], + out buffer0[6]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 2), + Load(ref values, inputStride, 5), + out buffer0[2], + out buffer0[5]); + + Av1ForwardTransformArithmetic.AddSubtract( + Load(ref values, inputStride, 3), + Load(ref values, inputStride, 4), + out buffer0[3], + out buffer1[4]); + + Av1ForwardTransformArithmetic.AddSubtract(buffer0[0], buffer0[3], out buffer1[0], out buffer1[3]); + Av1ForwardTransformArithmetic.AddSubtract(buffer0[1], buffer0[2], out buffer1[1], out buffer1[2]); + Av1ForwardTransformArithmetic.Butterfly( + -cospi[32], + cospi[32], + buffer0[5], + buffer0[6], + out buffer1[5], + out buffer1[6], + cosBit, + in rounding); + + // Stage 3 completes the even half directly in coefficient order and prepares the four remaining odd terms. + Av1ForwardTransformArithmetic.Butterfly( + cospi[32], + cospi[32], + buffer1[0], + buffer1[1], + out TValue output0, + out TValue output4, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.Butterfly( + cospi[16], + cospi[48], + buffer1[3], + buffer1[2], + out TValue output2, + out TValue output6, + cosBit, + in rounding); + + Av1ForwardTransformArithmetic.AddSubtract(buffer1[4], buffer1[5], out buffer0[4], out buffer0[5]); + Av1ForwardTransformArithmetic.AddSubtract(buffer1[7], buffer1[6], out buffer0[7], out buffer0[6]); - // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. - Av1ForwardTransformArithmetic.Butterfly(cospi[32], cospi[32], step[0], step[1], out output[0], out output[1], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[16], cospi[48], step[3], step[2], out output[2], out output[3], cosBit, in rounding); - Av1ForwardTransformArithmetic.AddSubtract(step[4], step[5], out output[4], out output[5]); - Av1ForwardTransformArithmetic.AddSubtract(step[7], step[6], out output[7], out output[6]); + // Highway fuses the final two stages because no intermediate value is reused after either rotation. + Av1ForwardTransformArithmetic.Butterfly( + cospi[8], + cospi[56], + buffer0[7], + buffer0[4], + out TValue output1, + out TValue output7, + cosBit, + in rounding); - // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. - step[0] = output[0]; - step[1] = output[1]; - step[2] = output[2]; - step[3] = output[3]; - Av1ForwardTransformArithmetic.Butterfly(cospi[8], cospi[56], output[7], output[4], out step[4], out step[7], cosBit, in rounding); - Av1ForwardTransformArithmetic.Butterfly(cospi[40], cospi[24], output[6], output[5], out step[5], out step[6], cosBit, in rounding); + Av1ForwardTransformArithmetic.Butterfly( + cospi[40], + cospi[24], + buffer0[6], + buffer0[5], + out TValue output5, + out TValue output3, + cosBit, + in rounding); - // Stage 5 permutes the staged values into ascending AV1 coefficient order. - output[0] = step[0]; - output[1] = step[4]; - output[2] = step[2]; - output[3] = step[6]; - output[4] = step[1]; - output[5] = step[5]; - output[6] = step[3]; - output[7] = step[7]; + Store(ref values, outputStride, 0, output0); + Store(ref values, outputStride, 1, output1); + Store(ref values, outputStride, 2, output2); + Store(ref values, outputStride, 3, output3); + Store(ref values, outputStride, 4, output4); + Store(ref values, outputStride, 5, output5); + Store(ref values, outputStride, 6, output6); + Store(ref values, outputStride, 7, output7); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs index 73c4cea43..67187adc2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformOperations.Identity.cs @@ -12,100 +12,124 @@ internal static partial class Av1ForwardTransformOperations /// Applies the four-point forward identity transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The unused transform-stage buffer. - /// The unused fixed-point precision. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. public static void Identity4( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Identity(ref input, ref output, ref step, cosBit, 4, 1, 0); + => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 4, 1, 0); /// /// Applies the eight-point forward identity transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The unused transform-stage buffer. - /// The unused fixed-point precision. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. public static void Identity8( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Identity(ref input, ref output, ref step, cosBit, 8, 0, 1); + => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 8, 0, 1); /// /// Applies the sixteen-point forward identity transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The unused transform-stage buffer. - /// The unused fixed-point precision. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. public static void Identity16( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Identity(ref input, ref output, ref step, cosBit, 16, 2, 0); + => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 16, 2, 0); /// /// Applies the thirty-two-point forward identity transform to every independent lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The unused transform-stage buffer. - /// The unused fixed-point precision. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. public static void Identity32( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Identity(ref input, ref output, ref step, cosBit, 32, 0, 2); + => Identity(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit, 32, 0, 2); /// - /// Applies the length-specific AV1 identity scaling to every transform value. + /// Applies the length-specific AV1 identity scaling directly to the strided transform block. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain values. - /// The frequency-domain values. - /// The unused transform-stage buffer. - /// The unused fixed-point precision. - /// The number of transform values. - /// The multiplier applied with the fixed-point square-root-of-two constant. - /// The direct left shift applied when square-root scaling is not required. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. + /// The fixed-point precision of the cosine constants. + /// The number of transform positions. + /// The square-root-of-two multiplier, or zero when power-of-two scaling applies. + /// The power-of-two scaling shift. private static void Identity( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit, int length, int sqrt2Scale, int leftShift) where TValue : struct { - _ = step; + _ = buffer0; + _ = buffer1; _ = cosBit; - // AV1 defines identity normalization by transform length: 4 and 16 use NewSqrt2 scaling, while 8 and 32 - // are exact powers of two. The same operation applies independently to each SIMD lane. + // AV1 defines identity normalization by transform length: 4 and 16 use sqrt(2) scaling, while 8 and 32 + // are exact powers of two. Applying it in place matches Highway's row-oriented identity kernels. for (int i = 0; i < length; i++) { - output[i] = sqrt2Scale != 0 + TValue input = Load(ref values, inputStride, i); + TValue output = sqrt2Scale != 0 ? Av1ForwardTransformArithmetic.MultiplyRound( - input[i], + input, sqrt2Scale * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) - : Av1ForwardTransformArithmetic.ShiftLeft(input[i], leftShift); + : Av1ForwardTransformArithmetic.ShiftLeft(input, leftShift); + + Store(ref values, outputStride, i, output); } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs index cd7dceca4..5216a8900 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity16Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Identity16Forward1dOperator : IAv1ForwardTransform1d { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Identity16(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Identity16(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs index 0b96e7bff..f15c0b868 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity32Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Identity32Forward1dOperator : IAv1ForwardTransform1d { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Identity32(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Identity32(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs index 751a8218e..89236df3d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity4Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Identity4Forward1dOperator : IAv1ForwardTransform1dO { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Identity4(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Identity4(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs index a330e6e8d..96a295221 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Identity8Forward1dOperator.cs @@ -10,10 +10,12 @@ internal readonly struct Av1Identity8Forward1dOperator : IAv1ForwardTransform1dO { /// public static void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct - => Av1ForwardTransformOperations.Identity8(ref input, ref output, ref step, cosBit); + => Av1ForwardTransformOperations.Identity8(ref values, inputStride, outputStride, ref buffer0, ref buffer1, cosBit); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs index 8cf0e5f72..6aa68233d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/IAv1ForwardTransform1dOperator.cs @@ -17,14 +17,18 @@ internal interface IAv1ForwardTransform1dOperator /// Transforms the independent axes stored in each value lane. /// /// The scalar or SIMD value containing the independent transform axes. - /// The spatial-domain transform values. - /// The frequency-domain transform values. - /// The fixed transform-stage buffer. + /// The first value in the strided transform block. + /// The byte distance between consecutive input positions. + /// The byte distance between consecutive output positions. + /// The first fixed transform-stage buffer. + /// The second fixed transform-stage buffer. /// The fixed-point precision of the cosine constants. public static abstract void Transform( - ref Av1TransformVector input, - ref Av1TransformVector output, - ref Av1TransformVector step, + ref byte values, + nint inputStride, + nint outputStride, + ref Av1TransformVector buffer0, + ref Av1TransformVector buffer1, int cosBit) where TValue : struct; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index b29624536..fc0ba961e 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -10,6 +10,9 @@ using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; +/// +/// Verifies AV1 forward transform arithmetic, dispatch, layout, and allocation behavior. +/// [Trait("Format", "Avif")] public class Av1ForwardTransformTests { @@ -31,6 +34,26 @@ public class Av1ForwardTransformTests public void OneDimensionalOperatorsMatchAcrossHardwareWidths() => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertOneDimensionalOperators, TransformConfigurations); + /// + /// Verifies every one-dimensional stage network against the independent analytical transform definition. + /// + [Fact] + public void OneDimensionalOperatorsMatchAnalyticalReference() + { + AssertOperatorAccuracy(Av1TransformType1d.Dct, 4); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 8); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 16); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 32); + AssertOperatorAccuracy(Av1TransformType1d.Dct, 64); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 4); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 8); + AssertOperatorAccuracy(Av1TransformType1d.Adst, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 4); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 8); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 16); + AssertOperatorAccuracy(Av1TransformType1d.Identity, 32); + } + /// /// Verifies every permitted size, type, and bit-depth combination against the direct scalar two-axis definition. /// @@ -114,9 +137,59 @@ public class Av1ForwardTransformTests } } + /// + /// Compares one integer stage network with the analytical transform used by the libaom forward-transform tests. + /// + /// The transform operator. + /// The analytical transform definition. + /// The transform length. + private static void AssertOperatorAccuracy(Av1TransformType1d transformType, int length) + where TOperator : struct, IAv1ForwardTransform1dOperator + { + const int cosBit = 13; + const int testBlockCount = 500; + const int maximumCoefficientError = 7; + Random random = new(0); + double[] referenceInput = new double[length]; + double[] referenceOutput = new double[length]; + Av1TransformVector values = default; + Av1TransformVector buffer0 = default; + Av1TransformVector buffer1 = default; + + for (int block = 0; block < testBlockCount; block++) + { + for (int index = 0; index < length; index++) + { + int input = random.Next(1024) - random.Next(1024); + values[index] = input; + referenceInput[index] = input; + } + + ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); + + TOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, cosBit); + Av1ReferenceTransform.ReferenceTransform1d(transformType, referenceInput, referenceOutput, length); + + // libaom permits seven integer coefficient units because each fixed-point butterfly rounds independently. + for (int index = 0; index < length; index++) + { + int expected = (int)Math.Round(referenceOutput[index], MidpointRounding.AwayFromZero); + int error = Math.Abs(values[index] - expected); + + Assert.True( + error <= maximumCoefficientError, + $"{typeof(TOperator).Name} coefficient {index}: expected {expected}, actual {values[index]}, error {error}."); + } + } + } + /// /// Compares one Int32 vector representation with the scalar Int32 stage network lane by lane. /// + /// The transform operator. + /// The SIMD value containing independent transform axes. + /// The transform length. + /// The fixed-point precision of the cosine constants. private static void AssertInt32Operator(int length, int cosBit) where TOperator : struct, IAv1ForwardTransform1dOperator where TVector : struct @@ -136,7 +209,10 @@ public class Av1ForwardTransformTests } } - TOperator.Transform(ref vectorValues, ref vectorBuffer0, ref vectorBuffer1, cosBit); + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf(); + + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); for (int lane = 0; lane < laneCount; lane++) { @@ -149,12 +225,14 @@ public class Av1ForwardTransformTests scalarValues[index] = GetInputValue(index, lane); } - TOperator.Transform(ref scalarValues, ref scalarBuffer0, ref scalarBuffer1, cosBit); + ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); + + TOperator.Transform(ref scalarValuesBase, sizeof(int), sizeof(int), ref scalarBuffer0, ref scalarBuffer1, cosBit); for (int index = 0; index < length; index++) { - ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorBuffer0[index]); - Assert.Equal(scalarBuffer0[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); + ref int firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } @@ -162,6 +240,10 @@ public class Av1ForwardTransformTests /// /// Compares one Int16 vector representation with the scalar Int16 stage network lane by lane. /// + /// The transform operator. + /// The SIMD value containing independent transform axes. + /// The transform length. + /// The fixed-point precision of the cosine constants. private static void AssertInt16Operator(int length, int cosBit) where TOperator : struct, IAv1ForwardTransform1dOperator where TVector : struct @@ -181,7 +263,10 @@ public class Av1ForwardTransformTests } } - TOperator.Transform(ref vectorValues, ref vectorBuffer0, ref vectorBuffer1, cosBit); + ref byte vectorValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref vectorValues); + nint vectorStride = System.Runtime.CompilerServices.Unsafe.SizeOf(); + + TOperator.Transform(ref vectorValuesBase, vectorStride, vectorStride, ref vectorBuffer0, ref vectorBuffer1, cosBit); for (int lane = 0; lane < laneCount; lane++) { @@ -194,12 +279,14 @@ public class Av1ForwardTransformTests scalarValues[index] = GetPackedInputValue(index, lane); } - TOperator.Transform(ref scalarValues, ref scalarBuffer0, ref scalarBuffer1, cosBit); + ref byte scalarValuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref scalarValues); + + TOperator.Transform(ref scalarValuesBase, sizeof(short), sizeof(short), ref scalarBuffer0, ref scalarBuffer1, cosBit); for (int index = 0; index < length; index++) { - ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorBuffer0[index]); - Assert.Equal(scalarBuffer0[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); + ref short firstLane = ref System.Runtime.CompilerServices.Unsafe.As(ref vectorValues[index]); + Assert.Equal(scalarValues[index], System.Runtime.CompilerServices.Unsafe.Add(ref firstLane, lane)); } } } @@ -231,6 +318,9 @@ public class Av1ForwardTransformTests /// /// Compares one complete transform with the direct scalar two-axis definition. /// + /// The compound transform type. + /// The transform-block dimensions. + /// The source sample bit depth. private static void AssertTwoDimensionalCase(Av1TransformType transformType, Av1TransformSize transformSize, int bitDepth) { int width = transformSize.GetWidth(); @@ -269,6 +359,10 @@ public class Av1ForwardTransformTests /// /// Selects the scalar reference column operator. /// + /// The spatial residual samples. + /// The number of input samples between rows. + /// The destination reference coefficients. + /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceColumn(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) { switch (config.TransformFunctionTypeColumn) @@ -315,6 +409,11 @@ public class Av1ForwardTransformTests /// /// Selects the scalar reference row operator. /// + /// The column transform operator. + /// The spatial residual samples. + /// The number of input samples between rows. + /// The destination reference coefficients. + /// The resolved transform functions, shifts, and axis orientation. private static void DispatchReferenceRow(Span input, int stride, Span output, ref Av1Transform2dFlipConfiguration config) where TColumnOperator : struct, IAv1ForwardTransform1dOperator { @@ -362,6 +461,12 @@ public class Av1ForwardTransformTests /// /// Applies the direct scalar column and row transform definition used as the layout and dispatch oracle. /// + /// The column transform operator. + /// The row transform operator. + /// The spatial residual samples. + /// The number of input samples between rows. + /// The destination reference coefficients. + /// The resolved transform functions, shifts, and axis orientation. private static void TransformReference( Span input, int stride, @@ -387,12 +492,14 @@ public class Av1ForwardTransformTests values[row] = input[(sourceRow * stride) + column] << config.Shift0; } - TColumnOperator.Transform(ref values, ref buffer0, ref buffer1, config.CosBitColumn); + ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); + + TColumnOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitColumn); int destinationColumn = config.FlipLeftToRight ? width - column - 1 : column; for (int row = 0; row < height; row++) { - intermediate[(row * width) + destinationColumn] = Av1Math.RoundShift(buffer0[row], -config.Shift1); + intermediate[(row * width) + destinationColumn] = Av1Math.RoundShift(values[row], -config.Shift1); } } @@ -405,11 +512,13 @@ public class Av1ForwardTransformTests values[column] = intermediate[(row * width) + column]; } - TRowOperator.Transform(ref values, ref buffer0, ref buffer1, config.CosBitRow); + ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); + + TRowOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, config.CosBitRow); for (int column = 0; column < outputWidth; column++) { - int value = Av1Math.RoundShift(buffer0[column], -config.Shift2); + int value = Av1Math.RoundShift(values[column], -config.Shift2); output[(row * outputWidth) + column] = normalizeRectangle ? Av1Transform1dMath.HalfButterfly(Av1Transform1dMath.NewSqrt2, value, 0, 0, Av1Transform1dMath.NewSqrt2Bits) : value; @@ -420,12 +529,18 @@ public class Av1ForwardTransformTests /// /// Gets a deterministic signed thirty-two-bit transform input. /// + /// The transform position. + /// The independent SIMD lane. + /// The deterministic input value. private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 8191) - 4095; /// /// Gets a deterministic signed sixteen-bit input including overflow-sensitive edge values. /// + /// The transform position. + /// The independent SIMD lane. + /// The deterministic packed input value. private static short GetPackedInputValue(int index, int lane) => (short)((index + lane) % 5 switch { @@ -439,6 +554,7 @@ public class Av1ForwardTransformTests /// /// Creates the complete normative transform matrix shared by the forward and inverse tests. /// + /// Every permitted transform type, transform size, and AV1 image bit depth. private static TheoryData CreateValidTransformCases() { TheoryData cases = []; diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 6f2fe1865..f6e12fd1d 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -562,11 +562,13 @@ public class Av1InverseTransformTests values[index] = input[index]; } - TForwardOperator.Transform(ref values, ref buffer0, ref buffer1, forwardConfig.CosBitColumn); + ref byte valuesBase = ref System.Runtime.CompilerServices.Unsafe.As, byte>(ref values); + + TForwardOperator.Transform(ref valuesBase, sizeof(int), sizeof(int), ref buffer0, ref buffer1, forwardConfig.CosBitColumn); for (int index = 0; index < length; index++) { - forward[index] = buffer0[index]; + forward[index] = values[index]; } TInverseOperator.Transform(forward, inverse, step, inverseConfig.CosBitColumn, inverseConfig.StageRangeColumn);